ls

en-es.tmx


from translate.storage.tmx import tmxfile

with open("en-es.tmx", "rb") as f:
    tmx_file = tmxfile(f, "en", "es")
    
source = []
target = []

for node in tmx_file.unit_iter():
    source.append( " ".join(node.source.split()) )
    target.append( " ".join(node.target.split()) )


print(len(source))
print(len(target))

len(source) == len(target)

405955
405955

True


source[:5]

['Thank you so much, Chris.',
 "And it's truly a great honor to have the opportunity to come to this stage twice; I'm extremely grateful.",
 'I have been blown away by this conference, and I want to thank all of you for the many nice comments about what I had to say the other night.',
 'And I say that sincerely, partly because (Mock sob) I need that.',
 '(Laughter) Put yourselves in my position.']


target[:5]

['Muchas gracias Chris.',
 'Y es en verdad un gran honor tener la oportunidad de venir a este escenario por segunda vez. Estoy extremadamente agradecido.',
 'He quedado conmovido por esta conferencia, y deseo agradecer a todos ustedes sus amables comentarios acerca de lo que tenía que decir la otra noche.',
 'Y digo eso sinceramente, en parte porque -- (Sollozos fingidos) -- ¡lo necesito!',
 '(Risas) ¡Pónganse en mi posición!']


import pandas as pd

data = pd.DataFrame({ "source": source,
                      "target": target})

data


data.isnull().any()

source    False
target    False
dtype: bool


import re
def simple_processing(text):
    
    text = str(text)
    text = text.lower()
    text = re.sub(r'[0-9]+', '', text) # eliminar numeros

    #normalize
    replacements = (
        ("á", "a"),
        ("é", "e"),
        ("í", "i"),
        ("ó", "o"),
        ("ú", "u"),
    )
    for a, b in replacements:
        text = text.replace(a, b).replace(a.upper(), b.upper())
    
    signos = ["_","-", "!", "¿", "?", "¡", "#",".", ",", ";", ":", "@", "*", "+", "(", ")", "/", "$", "\n", '•',"'"] + ['xxxx', 'xxx', 'xx']
    for i in signos:
        if i == "'":
            text = text.replace(i, "")
        else :
            text = text.replace(i, " ")
        
    return " ".join(text.split())


data["source"] = data["source"].apply(simple_processing)
data["target"] = data["target"].apply(simple_processing)

data


data.isnull().any()

source    False
target    False
dtype: bool


!mkdir Spanish_SBERT


!mkdir Spanish_SBERT\datasets


ls Spanish_SBERT

 Volume in drive C is NVMe
 Volume Serial Number is 80AD-482E

 Directory of C:\Users\josep\Documents\Python Scripts\SBERT\Spanish_SBERT

11/17/2022  07:56 PM    <DIR>          .
11/17/2022  07:56 PM    <DIR>          ..
11/17/2022  07:56 PM    <DIR>          datasets
               0 File(s)              0 bytes
               3 Dir(s)  319,105,437,696 bytes free


ls Spanish_SBERT\datasets

 Volume in drive C is NVMe
 Volume Serial Number is 80AD-482E

 Directory of C:\Users\josep\Documents\Python Scripts\SBERT\Spanish_SBERT\datasets

11/17/2022  07:56 PM    <DIR>          .
11/17/2022  07:56 PM    <DIR>          ..
               0 File(s)              0 bytes
               2 Dir(s)  319,105,372,160 bytes free


for i in range(5):
    data = data.sample( frac = 1 , ignore_index = True)

n_95 = int(data.shape[0]*0.95)
n_95

385657


data_train = data.iloc[ : n_95 , :]
data_dev   = data.iloc[ n_95 : , :]


data_dev.shape

(20298, 2)


ns = [15,25,50,75,100]

for n in ns:
    data_sample = data_train.sample(frac = n/100).reset_index(drop=True).dropna()
    data_sample.to_csv("Spanish_SBERT/datasets/en-es-tsv_train_sample_{}.tsv".format(n), sep = "\t", index = False)

data_dev.to_csv("Spanish_SBERT/datasets/en-es-tsv_dev.tsv", sep = "\t", index = False)


ls Spanish_SBERT\datasets

 Volume in drive C is NVMe
 Volume Serial Number is 80AD-482E

 Directory of C:\Users\josep\Documents\Python Scripts\SBERT\Spanish_SBERT\datasets

11/17/2022  07:56 PM    <DIR>          .
11/17/2022  07:56 PM    <DIR>          ..
11/17/2022  07:56 PM         3,903,902 en-es-tsv_dev.tsv
11/17/2022  07:56 PM        73,691,688 en-es-tsv_train_sample_100.tsv
11/17/2022  07:56 PM        11,059,859 en-es-tsv_train_sample_15.tsv
11/17/2022  07:56 PM        18,380,520 en-es-tsv_train_sample_25.tsv
11/17/2022  07:56 PM        36,859,594 en-es-tsv_train_sample_50.tsv
11/17/2022  07:56 PM        55,222,448 en-es-tsv_train_sample_75.tsv
               6 File(s)    199,118,011 bytes
               2 Dir(s)  318,906,101,760 bytes free


from sentence_transformers import SentenceTransformer, LoggingHandler, models, evaluation, losses
from torch.utils.data import DataLoader
from sentence_transformers.datasets import ParallelSentencesDataset
from datetime import datetime

import os
import logging
import sentence_transformers.util
import csv
import gzip
from tqdm.autonotebook import tqdm
import numpy as np
import zipfile
import io
import pandas as pd

C:\Users\josep\anaconda3\envs\nlp-39\lib\site-packages\tqdm\auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm


logging.basicConfig(format='%(asctime)s - %(message)s',
                    datefmt='%Y-%m-%d %H:%M:%S',
                    level=logging.INFO,
                    handlers=[LoggingHandler()])
logger = logging.getLogger(__name__)


#teacher_model_name = 'paraphrase-distilroberta-base-v2'   #Our monolingual teacher model, we want to convert to multiple languages
#student_model_name = 'xlm-roberta-base'       #Multilingual base model we use to imitate the teacher model

teacher_model_name = 'all-mpnet-base-v2' #768 dim   #Our monolingual teacher model, we want to convert to multiple languages
student_model_name = 'Spanish_SBERT/modelos_entrenados/Spanish_SBERT_100'       #Multilingual base model we use to imitate the teacher model

max_seq_length = 128                #Student model max. lengths for inputs (number of word pieces)
train_batch_size = 64               #Batch size for training
inference_batch_size = 64           #Batch size at inference
max_sentences_per_language = 500000 #Maximum number of  parallel sentences for training
train_max_sentence_length = 250     #Maximum length (characters) for parallel training sentences

#num_epochs = 5                       #Train for x epochs
num_epochs = 10                       #Train for x epochs
num_warmup_steps = 10000             #Warumup steps

num_evaluation_steps = 1000          #Evaluate performance after every xxxx steps
dev_sentences = 1000                 #Number of parallel sentences to be used for development


# Define the language codes you would like to extend the model to
source_languages = set(['en'])                      # Our teacher model accepts English (en) sentences
target_languages = set(['es'])    # We want to extend the model to these new languages. For language codes, see the header of the train file


output_path = "output/make-multilingual-"+"-".join(sorted(list(source_languages))+sorted(list(target_languages)))+"-"+datetime.now().strftime("%Y-%m-%d_%H-%M-%S")


# This function downloads a corpus if it does not exist
def download_corpora(filepaths):
    if not isinstance(filepaths, list):
        filepaths = [filepaths]

    for filepath in filepaths:
        if not os.path.exists(filepath):
            print(filepath, "does not exists. Try to download from server")
            filename = os.path.basename(filepath)
            url = "https://sbert.net/datasets/" + filename
            sentence_transformers.util.http_get(url, filepath)
        else :
            print(filepath, "found succesfully")


# Here we define train train and dev corpora
train_corpus = "Spanish_SBERT/datasets/en-es-tsv_train_sample_100.tsv"         # Transcripts of TED talks, crawled 2020
dev_corpus   = "Spanish_SBERT/datasets/en-es-tsv_dev.tsv"

sts_corpus = "Spanish_SBERT/datasets/STS2017-extended.zip"     # Extended STS2017 dataset for more languages
parallel_sentences_folder = "Spanish_SBERT/parallel-sentences/"

# Check if the file exists. If not, they are downloaded
download_corpora([train_corpus, sts_corpus])


os.makedirs( parallel_sentences_folder, exist_ok = True )
train_files = []
dev_files = []
files_to_create = []

Spanish_SBERT/datasets/en-es-tsv_train_sample_100.tsv found succesfully
Spanish_SBERT/datasets/STS2017-extended.zip found succesfully


'''
for source_lang in source_languages:
    for target_lang in target_languages:
        output_filename_train = os.path.join(parallel_sentences_folder, "TED2020-{}-{}-train.tsv.gz".format(source_lang, target_lang))
        output_filename_dev = os.path.join(parallel_sentences_folder, "TED2020-{}-{}-dev.tsv.gz".format(source_lang, target_lang))
        train_files.append(output_filename_train)
        dev_files.append(output_filename_dev)
        if not os.path.exists(output_filename_train) or not os.path.exists(output_filename_dev):
            files_to_create.append({'src_lang': source_lang, 'trg_lang': target_lang,
                                    'fTrain': gzip.open(output_filename_train, 'wt', encoding='utf8'),
                                    'fDev': gzip.open(output_filename_dev, 'wt', encoding='utf8'),
                                    'devCount': 0
                                    })
'''

'\nfor source_lang in source_languages:\n    for target_lang in target_languages:\n        output_filename_train = os.path.join(parallel_sentences_folder, "TED2020-{}-{}-train.tsv.gz".format(source_lang, target_lang))\n        output_filename_dev = os.path.join(parallel_sentences_folder, "TED2020-{}-{}-dev.tsv.gz".format(source_lang, target_lang))\n        train_files.append(output_filename_train)\n        dev_files.append(output_filename_dev)\n        if not os.path.exists(output_filename_train) or not os.path.exists(output_filename_dev):\n            files_to_create.append({\'src_lang\': source_lang, \'trg_lang\': target_lang,\n                                    \'fTrain\': gzip.open(output_filename_train, \'wt\', encoding=\'utf8\'),\n                                    \'fDev\': gzip.open(output_filename_dev, \'wt\', encoding=\'utf8\'),\n                                    \'devCount\': 0\n                                    })\n'


'''
if len(files_to_create) > 0:
    print("Parallel sentences files {} do not exist. Create these files now".format(", ".join(map(lambda x: x['src_lang']+"-"+x['trg_lang'], files_to_create))))
    with gzip.open(train_corpus, 'rt', encoding='utf8') as fIn:
        reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
        for line in tqdm(reader, desc="Sentences"):
            for outfile in files_to_create:
                src_text = line[outfile['src_lang']].strip()
                trg_text = line[outfile['trg_lang']].strip()

                if src_text != "" and trg_text != "":
                    if outfile['devCount'] < dev_sentences:
                        outfile['devCount'] += 1
                        fOut = outfile['fDev']
                    else:
                        fOut = outfile['fTrain']

                    fOut.write("{}\t{}\n".format(src_text, trg_text))

    for outfile in files_to_create:
        outfile['fTrain'].close()
        outfile['fDev'].close()
'''

'\nif len(files_to_create) > 0:\n    print("Parallel sentences files {} do not exist. Create these files now".format(", ".join(map(lambda x: x[\'src_lang\']+"-"+x[\'trg_lang\'], files_to_create))))\n    with gzip.open(train_corpus, \'rt\', encoding=\'utf8\') as fIn:\n        reader = csv.DictReader(fIn, delimiter=\'\t\', quoting=csv.QUOTE_NONE)\n        for line in tqdm(reader, desc="Sentences"):\n            for outfile in files_to_create:\n                src_text = line[outfile[\'src_lang\']].strip()\n                trg_text = line[outfile[\'trg_lang\']].strip()\n\n                if src_text != "" and trg_text != "":\n                    if outfile[\'devCount\'] < dev_sentences:\n                        outfile[\'devCount\'] += 1\n                        fOut = outfile[\'fDev\']\n                    else:\n                        fOut = outfile[\'fTrain\']\n\n                    fOut.write("{}\t{}\n".format(src_text, trg_text))\n\n    for outfile in files_to_create:\n        outfile[\'fTrain\'].close()\n        outfile[\'fDev\'].close()\n'


train_files.append(train_corpus)
dev_files.append(dev_corpus)

######## Start the extension of the teacher model to multiple languages ########
logger.info("Load teacher model")
teacher_model = SentenceTransformer(teacher_model_name)


logger.info("Create student model from scratch")
word_embedding_model = models.Transformer(student_model_name, max_seq_length=max_seq_length)
# Apply mean pooling to get one fixed sized sentence vector
pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())
student_model = SentenceTransformer(modules=[word_embedding_model, pooling_model])


###### Read Parallel Sentences Dataset ######
train_data = ParallelSentencesDataset(student_model=student_model, teacher_model=teacher_model, batch_size=inference_batch_size, use_embedding_cache=True)
for train_file in train_files:
    train_data.load_data(train_file, max_sentences=max_sentences_per_language, max_sentence_length=train_max_sentence_length)

train_dataloader = DataLoader(train_data, shuffle=True, batch_size=train_batch_size)
train_loss = losses.MSELoss(model=student_model)

2022-11-22 20:59:39 - Load teacher model
2022-11-22 20:59:39 - Load pretrained SentenceTransformer: all-mpnet-base-v2
2022-11-22 20:59:40 - Use pytorch device: cuda
2022-11-22 20:59:40 - Create student model from scratch
2022-11-22 20:59:41 - Use pytorch device: cuda
2022-11-22 20:59:41 - Load Spanish_SBERT/datasets/en-es-tsv_train_sample_100.tsv


#### Evaluate cross-lingual performance on different tasks #####
#evaluators = []         #evaluators has a list of different evaluator classes we call periodically

#for dev_file in dev_files:
#    logger.info("Create evaluator for " + dev_file)
#    src_sentences = []
#    trg_sentences = []
#    with gzip.open(dev_file, 'rt', encoding='utf8') as fIn:
#        for line in fIn:
#            splits = line.strip().split('\t')
#            if splits[0] != "" and splits[1] != "":
#                src_sentences.append(splits[0])
#                trg_sentences.append(splits[1])


#    #Mean Squared Error (MSE) measures the (euclidean) distance between teacher and student embeddings
#    dev_mse = evaluation.MSEEvaluator(src_sentences, trg_sentences, name=os.path.basename(dev_file), teacher_model=teacher_model, batch_size=inference_batch_size)
#    evaluators.append(dev_mse)

#    # TranslationEvaluator computes the embeddings for all parallel sentences. It then check if the embedding of source[i] is the closest to target[i] out of all available target sentences
#    dev_trans_acc = evaluation.TranslationEvaluator(src_sentences, trg_sentences, name=os.path.basename(dev_file),batch_size=inference_batch_size)
#    evaluators.append(dev_trans_acc)
    
evaluators = []         #evaluators has a list of different evaluator classes we call periodically

dev_file = dev_files[0]

logger.info("Create evaluator for " + dev_file)
src_sentences = []
trg_sentences = []

dev_data = pd.read_csv( dev_files[0] , sep = '\t' )
for source, target in zip( dev_data['source'], dev_data['target'] ):
    src_sentences.append(source)
    trg_sentences.append(target)



#Mean Squared Error (MSE) measures the (euclidean) distance between teacher and student embeddings
dev_mse = evaluation.MSEEvaluator(src_sentences, trg_sentences, name=os.path.basename(dev_file), teacher_model=teacher_model, batch_size=inference_batch_size)
evaluators.append(dev_mse)

# TranslationEvaluator computes the embeddings for all parallel sentences. It then check if the embedding of source[i] is the closest to target[i] out of all available target sentences
dev_trans_acc = evaluation.TranslationEvaluator(src_sentences, trg_sentences, name=os.path.basename(dev_file),batch_size=inference_batch_size)
evaluators.append(dev_trans_acc)

2022-11-22 20:59:43 - Create evaluator for Spanish_SBERT/datasets/en-es-tsv_dev.tsv


dev_data = pd.read_csv( dev_files[0] , sep = '\t' )
print(dev_data.shape)
dev_data.head()

(20298, 2)


train_data = pd.read_csv( train_files[0] , sep = '\t' )
print(train_data.shape)
train_data.head()

(385657, 2)


dev_files[0]

'Spanish_SBERT/datasets/en-es-tsv_dev.tsv'


##### Read cross-lingual Semantic Textual Similarity (STS) data ####
all_languages = list(set(list(source_languages)+list(target_languages)))
sts_data = {}

#Open the ZIP File of STS2017-extended.zip and check for which language combinations we have STS data
with zipfile.ZipFile(sts_corpus) as zip:
    filelist = zip.namelist()
    sts_files = []

    for i in range(len(all_languages)):
        for j in range(i, len(all_languages)):
            lang1 = all_languages[i]
            lang2 = all_languages[j]
            filepath = 'STS2017-extended/STS.{}-{}.txt'.format(lang1, lang2)
            if filepath not in filelist:
                lang1, lang2 = lang2, lang1
                filepath = 'STS2017-extended/STS.{}-{}.txt'.format(lang1, lang2)

            if filepath in filelist:
                filename = os.path.basename(filepath)
                sts_data[filename] = {'sentences1': [], 'sentences2': [], 'scores': []}

                fIn = zip.open(filepath)
                for line in io.TextIOWrapper(fIn, 'utf8'):
                    sent1, sent2, score = line.strip().split("\t")
                    score = float(score)
                    sts_data[filename]['sentences1'].append(sent1)
                    sts_data[filename]['sentences2'].append(sent2)
                    sts_data[filename]['scores'].append(score)

for filename, data in sts_data.items():
    test_evaluator = evaluation.EmbeddingSimilarityEvaluator(data['sentences1'], data['sentences2'], data['scores'], batch_size=inference_batch_size, name=filename, show_progress_bar=False)
    evaluators.append(test_evaluator)


train_dataloader

<torch.utils.data.dataloader.DataLoader at 0x180491eae80>

ls

 Volume in drive C is NVMe
 Volume Serial Number is 80AD-482E

 Directory of C:\Users\josep\Documents\Python Scripts\SBERT

11/22/2022  08:59 PM    <DIR>          .
11/17/2022  03:47 PM    <DIR>          ..
11/22/2022  08:57 PM    <DIR>          .ipynb_checkpoints
12/03/2020  12:58 AM       122,976,777 en-es.tmx
11/14/2022  11:13 AM        32,737,526 en-es.tmx.gz
11/17/2022  08:01 PM    <DIR>          output
11/17/2022  07:57 PM    <DIR>          Spanish_SBERT
11/17/2022  03:49 PM           138,165 spanish_SBERT.ipynb
11/21/2022  12:20 AM            58,577 Spanish_SBERT_new - Copy.ipynb
11/22/2022  03:24 PM        25,118,608 Spanish_SBERT_new.ipynb
11/22/2022  08:59 PM        25,118,897 Spanish_SBERT_new_v2.ipynb
11/22/2022  08:36 PM            28,965 Untitled.ipynb
               7 File(s)    206,177,515 bytes
               5 Dir(s)  276,518,797,312 bytes free


!mkdir Spanish_SBERT\modelos_entrenados
!mkdir Spanish_SBERT\modelos_entrenados\Spanish_SBERT_100

A subdirectory or file Spanish_SBERT\modelos_entrenados already exists.
A subdirectory or file Spanish_SBERT\modelos_entrenados\Spanish_SBERT_100 already exists.


from time import time

t1 = time()

# Train the model
student_model.fit(train_objectives=[(train_dataloader, train_loss)],
          evaluator=evaluation.SequentialEvaluator(evaluators, main_score_function=lambda scores: np.mean(scores)),
          epochs=num_epochs,
          warmup_steps=num_warmup_steps,
          evaluation_steps=num_evaluation_steps,
          output_path=output_path,
          save_best_model=True,
          #optimizer_params= {'lr': 2e-5, 'eps': 1e-6, 'correct_bias': False}
          optimizer_params= {'lr': 2e-5, 'eps': 1e-6}
          )

t2 = time()

Epoch:   0%|          | 0/10 [00:00<?, ?it/s]
Iteration:   0%|          | 0/11591 [00:00<?, ?it/s]
Iteration:   0%|          | 1/11591 [00:00<1:44:03,  1.86it/s]
Iteration:   0%|          | 2/11591 [00:00<1:14:22,  2.60it/s]
Iteration:   0%|          | 3/11591 [00:00<1:00:57,  3.17it/s]
Iteration:   0%|          | 4/11591 [00:01<52:49,  3.66it/s]  
Iteration:   0%|          | 5/11591 [00:01<58:07,  3.32it/s]
Iteration:   0%|          | 6/11591 [00:01<53:06,  3.64it/s]
Iteration:   0%|          | 7/11591 [00:01<49:44,  3.88it/s]
Iteration:   0%|          | 8/11591 [00:02<49:30,  3.90it/s]
Iteration:   0%|          | 9/11591 [00:02<48:16,  4.00it/s]
Iteration:   0%|          | 10/11591 [00:02<47:03,  4.10it/s]
Iteration:   0%|          | 11/11591 [00:02<45:42,  4.22it/s]
Iteration:   0%|          | 12/11591 [00:02<45:28,  4.24it/s]
Iteration:   0%|          | 13/11591 [00:03<44:38,  4.32it/s]
Iteration:   0%|          | 14/11591 [00:03<43:51,  4.40it/s]
Iteration:   0%|          | 15/11591 [00:03<44:18,  4.35it/s]
Iteration:   0%|          | 16/11591 [00:03<43:08,  4.47it/s]
Iteration:   0%|          | 17/11591 [00:03<42:20,  4.56it/s]
Iteration:   0%|          | 18/11591 [00:04<42:41,  4.52it/s]
Iteration:   0%|          | 19/11591 [00:04<41:56,  4.60it/s]
Iteration:   0%|          | 20/11591 [00:04<41:34,  4.64it/s]
Iteration:   0%|          | 21/11591 [00:04<41:52,  4.60it/s]
Iteration:   0%|          | 22/11591 [00:04<41:35,  4.64it/s]
Iteration:   0%|          | 23/11591 [00:05<41:14,  4.67it/s]
Iteration:   0%|          | 24/11591 [00:05<41:41,  4.62it/s]
Iteration:   0%|          | 25/11591 [00:05<41:02,  4.70it/s]
Iteration:   0%|          | 26/11591 [00:05<40:36,  4.75it/s]
Iteration:   0%|          | 27/11591 [00:05<40:55,  4.71it/s]
Iteration:   0%|          | 28/11591 [00:06<40:39,  4.74it/s]
Iteration:   0%|          | 29/11591 [00:06<40:09,  4.80it/s]
Iteration:   0%|          | 30/11591 [00:06<40:39,  4.74it/s]
Iteration:   0%|          | 31/11591 [00:06<40:26,  4.76it/s]
Iteration:   0%|          | 32/11591 [00:06<40:03,  4.81it/s]
Iteration:   0%|          | 33/11591 [00:07<40:00,  4.81it/s]
Iteration:   0%|          | 34/11591 [00:07<40:23,  4.77it/s]
Iteration:   0%|          | 35/11591 [00:07<40:10,  4.79it/s]
Iteration:   0%|          | 36/11591 [00:07<39:53,  4.83it/s]
Iteration:   0%|          | 37/11591 [00:08<40:30,  4.75it/s]
Iteration:   0%|          | 38/11591 [00:08<40:14,  4.79it/s]
Iteration:   0%|          | 39/11591 [00:08<39:58,  4.82it/s]
Iteration:   0%|          | 40/11591 [00:08<40:29,  4.75it/s]
Iteration:   0%|          | 41/11591 [00:08<40:00,  4.81it/s]
Iteration:   0%|          | 42/11591 [00:09<39:54,  4.82it/s]
Iteration:   0%|          | 43/11591 [00:09<40:24,  4.76it/s]
Iteration:   0%|          | 44/11591 [00:09<40:13,  4.78it/s]
Iteration:   0%|          | 45/11591 [00:09<40:06,  4.80it/s]
Iteration:   0%|          | 46/11591 [00:09<40:27,  4.76it/s]
Iteration:   0%|          | 47/11591 [00:10<40:09,  4.79it/s]
Iteration:   0%|          | 48/11591 [00:10<39:45,  4.84it/s]
Iteration:   0%|          | 49/11591 [00:10<40:15,  4.78it/s]
Iteration:   0%|          | 50/11591 [00:10<40:09,  4.79it/s]
Iteration:   0%|          | 51/11591 [00:10<40:01,  4.81it/s]
Iteration:   0%|          | 52/11591 [00:11<40:23,  4.76it/s]
Iteration:   0%|          | 53/11591 [00:11<40:13,  4.78it/s]
Iteration:   0%|          | 54/11591 [00:11<40:00,  4.81it/s]
Iteration:   0%|          | 55/11591 [00:11<39:48,  4.83it/s]
Iteration:   0%|          | 56/11591 [00:11<40:11,  4.78it/s]
Iteration:   0%|          | 57/11591 [00:12<40:03,  4.80it/s]
Iteration:   1%|          | 58/11591 [00:12<40:05,  4.79it/s]
Iteration:   1%|          | 59/11591 [00:12<40:36,  4.73it/s]
Iteration:   1%|          | 60/11591 [00:12<40:19,  4.77it/s]
Iteration:   1%|          | 61/11591 [00:13<40:18,  4.77it/s]
Iteration:   1%|          | 62/11591 [00:13<40:22,  4.76it/s]
Iteration:   1%|          | 63/11591 [00:13<40:11,  4.78it/s]
Iteration:   1%|          | 64/11591 [00:13<39:50,  4.82it/s]
Iteration:   1%|          | 65/11591 [00:13<40:08,  4.79it/s]
Iteration:   1%|          | 66/11591 [00:14<39:55,  4.81it/s]
Iteration:   1%|          | 67/11591 [00:14<39:41,  4.84it/s]
Iteration:   1%|          | 68/11591 [00:14<40:01,  4.80it/s]
Iteration:   1%|          | 69/11591 [00:14<39:46,  4.83it/s]
Iteration:   1%|          | 70/11591 [00:14<39:34,  4.85it/s]
Iteration:   1%|          | 71/11591 [00:15<40:13,  4.77it/s]
Iteration:   1%|          | 72/11591 [00:15<39:59,  4.80it/s]
Iteration:   1%|          | 73/11591 [00:15<39:50,  4.82it/s]
Iteration:   1%|          | 74/11591 [00:15<40:25,  4.75it/s]
Iteration:   1%|          | 75/11591 [00:15<39:55,  4.81it/s]
Iteration:   1%|          | 76/11591 [00:16<39:37,  4.84it/s]
Iteration:   1%|          | 77/11591 [00:16<39:30,  4.86it/s]
Iteration:   1%|          | 78/11591 [00:16<40:10,  4.78it/s]
Iteration:   1%|          | 79/11591 [00:16<40:10,  4.78it/s]
Iteration:   1%|          | 80/11591 [00:16<40:15,  4.77it/s]
Iteration:   1%|          | 81/11591 [00:17<40:30,  4.74it/s]
Iteration:   1%|          | 82/11591 [00:17<40:22,  4.75it/s]
Iteration:   1%|          | 83/11591 [00:17<40:00,  4.79it/s]
Iteration:   1%|          | 84/11591 [00:17<40:19,  4.76it/s]
Iteration:   1%|          | 85/11591 [00:18<39:57,  4.80it/s]
Iteration:   1%|          | 86/11591 [00:18<39:48,  4.82it/s]
Iteration:   1%|          | 87/11591 [00:18<40:22,  4.75it/s]
Iteration:   1%|          | 88/11591 [00:18<40:10,  4.77it/s]
Iteration:   1%|          | 89/11591 [00:18<39:46,  4.82it/s]
Iteration:   1%|          | 90/11591 [00:19<40:16,  4.76it/s]
Iteration:   1%|          | 91/11591 [00:19<40:01,  4.79it/s]
Iteration:   1%|          | 92/11591 [00:19<39:29,  4.85it/s]
Iteration:   1%|          | 93/11591 [00:19<39:45,  4.82it/s]
Iteration:   1%|          | 94/11591 [00:19<39:35,  4.84it/s]
Iteration:   1%|          | 95/11591 [00:20<39:41,  4.83it/s]
Iteration:   1%|          | 96/11591 [00:20<39:35,  4.84it/s]
Iteration:   1%|          | 97/11591 [00:20<40:07,  4.77it/s]
Iteration:   1%|          | 98/11591 [00:20<40:05,  4.78it/s]
Iteration:   1%|          | 99/11591 [00:20<39:58,  4.79it/s]
Iteration:   1%|          | 100/11591 [00:21<40:11,  4.76it/s]
Iteration:   1%|          | 101/11591 [00:21<39:39,  4.83it/s]
Iteration:   1%|          | 102/11591 [00:21<39:28,  4.85it/s]
Iteration:   1%|          | 103/11591 [00:21<39:51,  4.80it/s]
Iteration:   1%|          | 104/11591 [00:21<40:00,  4.79it/s]
Iteration:   1%|          | 105/11591 [00:22<39:41,  4.82it/s]
Iteration:   1%|          | 106/11591 [00:22<40:13,  4.76it/s]
Iteration:   1%|          | 107/11591 [00:22<39:54,  4.80it/s]
Iteration:   1%|          | 108/11591 [00:22<39:55,  4.79it/s]
Iteration:   1%|          | 109/11591 [00:23<40:19,  4.75it/s]
Iteration:   1%|          | 110/11591 [00:23<39:52,  4.80it/s]
Iteration:   1%|          | 111/11591 [00:23<39:41,  4.82it/s]
Iteration:   1%|          | 112/11591 [00:23<39:59,  4.78it/s]
Iteration:   1%|          | 113/11591 [00:23<39:39,  4.82it/s]
Iteration:   1%|          | 114/11591 [00:23<39:19,  4.86it/s]
Iteration:   1%|          | 115/11591 [00:24<39:18,  4.87it/s]
Iteration:   1%|          | 116/11591 [00:24<39:47,  4.81it/s]
Iteration:   1%|          | 117/11591 [00:24<39:41,  4.82it/s]
Iteration:   1%|          | 118/11591 [00:24<39:35,  4.83it/s]
Iteration:   1%|          | 119/11591 [00:25<39:46,  4.81it/s]
Iteration:   1%|          | 120/11591 [00:25<39:35,  4.83it/s]
Iteration:   1%|          | 121/11591 [00:25<39:10,  4.88it/s]
Iteration:   1%|          | 122/11591 [00:25<39:38,  4.82it/s]
Iteration:   1%|          | 123/11591 [00:25<39:26,  4.85it/s]
Iteration:   1%|          | 124/11591 [00:26<39:17,  4.86it/s]
Iteration:   1%|          | 125/11591 [00:26<39:55,  4.79it/s]
Iteration:   1%|          | 126/11591 [00:26<39:59,  4.78it/s]
Iteration:   1%|          | 127/11591 [00:26<39:36,  4.82it/s]
Iteration:   1%|          | 128/11591 [00:26<40:09,  4.76it/s]
Iteration:   1%|          | 129/11591 [00:27<39:53,  4.79it/s]
Iteration:   1%|          | 130/11591 [00:27<39:39,  4.82it/s]
Iteration:   1%|          | 131/11591 [00:27<39:14,  4.87it/s]
Iteration:   1%|          | 132/11591 [00:27<39:42,  4.81it/s]
Iteration:   1%|          | 133/11591 [00:27<39:32,  4.83it/s]
Iteration:   1%|          | 134/11591 [00:28<39:43,  4.81it/s]
Iteration:   1%|          | 135/11591 [00:28<40:04,  4.77it/s]
Iteration:   1%|          | 136/11591 [00:28<39:56,  4.78it/s]
Iteration:   1%|          | 137/11591 [00:28<39:50,  4.79it/s]
Iteration:   1%|          | 138/11591 [00:29<40:07,  4.76it/s]
Iteration:   1%|          | 139/11591 [00:29<39:53,  4.78it/s]
Iteration:   1%|          | 140/11591 [00:29<39:38,  4.81it/s]
Iteration:   1%|          | 141/11591 [00:29<40:02,  4.77it/s]
Iteration:   1%|          | 142/11591 [00:29<39:53,  4.78it/s]
Iteration:   1%|          | 143/11591 [00:30<39:27,  4.84it/s]
Iteration:   1%|          | 144/11591 [00:30<39:27,  4.83it/s]
Iteration:   1%|▏         | 145/11591 [00:30<38:57,  4.90it/s]
Iteration:   1%|▏         | 146/11591 [00:30<38:39,  4.93it/s]
Iteration:   1%|▏         | 147/11591 [00:30<39:10,  4.87it/s]
Iteration:   1%|▏         | 148/11591 [00:31<39:09,  4.87it/s]
Iteration:   1%|▏         | 149/11591 [00:31<39:04,  4.88it/s]
Iteration:   1%|▏         | 150/11591 [00:31<39:25,  4.84it/s]
Iteration:   1%|▏         | 151/11591 [00:31<39:23,  4.84it/s]
Iteration:   1%|▏         | 152/11591 [00:31<39:39,  4.81it/s]
Iteration:   1%|▏         | 153/11591 [00:32<39:27,  4.83it/s]
Iteration:   1%|▏         | 154/11591 [00:32<40:08,  4.75it/s]
Iteration:   1%|▏         | 155/11591 [00:32<40:21,  4.72it/s]
Iteration:   1%|▏         | 156/11591 [00:32<40:37,  4.69it/s]
Iteration:   1%|▏         | 157/11591 [00:33<40:43,  4.68it/s]
Iteration:   1%|▏         | 158/11591 [00:33<40:31,  4.70it/s]
Iteration:   1%|▏         | 159/11591 [00:33<40:19,  4.72it/s]
Iteration:   1%|▏         | 160/11591 [00:33<40:18,  4.73it/s]
Iteration:   1%|▏         | 161/11591 [00:33<39:58,  4.77it/s]
Iteration:   1%|▏         | 162/11591 [00:34<39:50,  4.78it/s]
Iteration:   1%|▏         | 163/11591 [00:34<40:18,  4.72it/s]
Iteration:   1%|▏         | 164/11591 [00:34<40:02,  4.76it/s]
Iteration:   1%|▏         | 165/11591 [00:34<39:48,  4.78it/s]
Iteration:   1%|▏         | 166/11591 [00:34<40:16,  4.73it/s]
Iteration:   1%|▏         | 167/11591 [00:35<40:01,  4.76it/s]
Iteration:   1%|▏         | 168/11591 [00:35<39:43,  4.79it/s]
Iteration:   1%|▏         | 169/11591 [00:35<40:12,  4.73it/s]
Iteration:   1%|▏         | 170/11591 [00:35<40:02,  4.75it/s]
Iteration:   1%|▏         | 171/11591 [00:35<39:55,  4.77it/s]
Iteration:   1%|▏         | 172/11591 [00:36<40:26,  4.71it/s]
Iteration:   1%|▏         | 173/11591 [00:36<40:04,  4.75it/s]
Iteration:   2%|▏         | 174/11591 [00:36<39:56,  4.77it/s]
Iteration:   2%|▏         | 175/11591 [00:36<39:42,  4.79it/s]
Iteration:   2%|▏         | 176/11591 [00:37<40:02,  4.75it/s]
Iteration:   2%|▏         | 177/11591 [00:37<39:46,  4.78it/s]
Iteration:   2%|▏         | 178/11591 [00:37<39:35,  4.81it/s]
Iteration:   2%|▏         | 179/11591 [00:37<40:04,  4.75it/s]
Iteration:   2%|▏         | 180/11591 [00:37<39:49,  4.78it/s]
Iteration:   2%|▏         | 181/11591 [00:38<39:32,  4.81it/s]
Iteration:   2%|▏         | 182/11591 [00:38<39:40,  4.79it/s]
Iteration:   2%|▏         | 183/11591 [00:38<39:27,  4.82it/s]
Iteration:   2%|▏         | 184/11591 [00:38<39:36,  4.80it/s]
Iteration:   2%|▏         | 185/11591 [00:38<40:11,  4.73it/s]
Iteration:   2%|▏         | 186/11591 [00:39<40:03,  4.74it/s]
Iteration:   2%|▏         | 187/11591 [00:39<39:43,  4.79it/s]
Iteration:   2%|▏         | 188/11591 [00:39<40:19,  4.71it/s]
Iteration:   2%|▏         | 189/11591 [00:39<40:09,  4.73it/s]
Iteration:   2%|▏         | 190/11591 [00:39<40:00,  4.75it/s]
Iteration:   2%|▏         | 191/11591 [00:40<40:13,  4.72it/s]
Iteration:   2%|▏         | 192/11591 [00:40<39:54,  4.76it/s]
Iteration:   2%|▏         | 193/11591 [00:40<39:36,  4.80it/s]
Iteration:   2%|▏         | 194/11591 [00:40<39:54,  4.76it/s]
Iteration:   2%|▏         | 195/11591 [00:40<39:47,  4.77it/s]
Iteration:   2%|▏         | 196/11591 [00:41<39:28,  4.81it/s]
Iteration:   2%|▏         | 197/11591 [00:41<39:06,  4.85it/s]
Iteration:   2%|▏         | 198/11591 [00:41<39:41,  4.78it/s]
Iteration:   2%|▏         | 199/11591 [00:41<39:25,  4.82it/s]
Iteration:   2%|▏         | 200/11591 [00:41<39:08,  4.85it/s]
Iteration:   2%|▏         | 201/11591 [00:42<39:37,  4.79it/s]
Iteration:   2%|▏         | 202/11591 [00:42<39:24,  4.82it/s]
Iteration:   2%|▏         | 203/11591 [00:42<39:07,  4.85it/s]
Iteration:   2%|▏         | 204/11591 [00:42<39:08,  4.85it/s]
Iteration:   2%|▏         | 205/11591 [00:43<39:11,  4.84it/s]
Iteration:   2%|▏         | 206/11591 [00:43<39:40,  4.78it/s]
Iteration:   2%|▏         | 207/11591 [00:43<40:13,  4.72it/s]
Iteration:   2%|▏         | 208/11591 [00:43<40:09,  4.72it/s]
Iteration:   2%|▏         | 209/11591 [00:43<40:09,  4.72it/s]
Iteration:   2%|▏         | 210/11591 [00:44<41:14,  4.60it/s]
Iteration:   2%|▏         | 211/11591 [00:44<41:22,  4.58it/s]
Iteration:   2%|▏         | 212/11591 [00:44<41:39,  4.55it/s]
Iteration:   2%|▏         | 213/11591 [00:45<42:10,  4.50it/s]
Iteration:   2%|▏         | 214/11591 [00:45<42:50,  4.43it/s]
Iteration:   2%|▏         | 215/11591 [00:45<42:35,  4.45it/s]
Iteration:   2%|▏         | 216/11591 [00:45<42:26,  4.47it/s]
Iteration:   2%|▏         | 217/11591 [00:45<42:45,  4.43it/s]
Iteration:   2%|▏         | 218/11591 [00:46<42:36,  4.45it/s]
Iteration:   2%|▏         | 219/11591 [00:46<42:06,  4.50it/s]
Iteration:   2%|▏         | 220/11591 [00:46<42:16,  4.48it/s]
Iteration:   2%|▏         | 221/11591 [00:46<42:03,  4.51it/s]
Iteration:   2%|▏         | 222/11591 [00:46<41:30,  4.56it/s]
Iteration:   2%|▏         | 223/11591 [00:47<41:42,  4.54it/s]
Iteration:   2%|▏         | 224/11591 [00:47<41:26,  4.57it/s]
Iteration:   2%|▏         | 225/11591 [00:47<40:51,  4.64it/s]
Iteration:   2%|▏         | 226/11591 [00:47<41:43,  4.54it/s]
Iteration:   2%|▏         | 227/11591 [00:48<41:25,  4.57it/s]
Iteration:   2%|▏         | 228/11591 [00:48<41:46,  4.53it/s]
Iteration:   2%|▏         | 229/11591 [00:48<42:24,  4.47it/s]
Iteration:   2%|▏         | 230/11591 [00:48<42:22,  4.47it/s]
Iteration:   2%|▏         | 231/11591 [00:49<41:53,  4.52it/s]
Iteration:   2%|▏         | 232/11591 [00:49<41:27,  4.57it/s]
Iteration:   2%|▏         | 233/11591 [00:49<42:26,  4.46it/s]
Iteration:   2%|▏         | 234/11591 [00:49<42:16,  4.48it/s]
Iteration:   2%|▏         | 235/11591 [00:49<42:37,  4.44it/s]
Iteration:   2%|▏         | 236/11591 [00:50<42:46,  4.42it/s]
Iteration:   2%|▏         | 237/11591 [00:50<43:10,  4.38it/s]
Iteration:   2%|▏         | 238/11591 [00:50<42:53,  4.41it/s]
Iteration:   2%|▏         | 239/11591 [00:50<43:13,  4.38it/s]
Iteration:   2%|▏         | 240/11591 [00:51<42:51,  4.41it/s]
Iteration:   2%|▏         | 241/11591 [00:51<42:28,  4.45it/s]
Iteration:   2%|▏         | 242/11591 [00:51<42:38,  4.44it/s]
Iteration:   2%|▏         | 243/11591 [00:51<42:44,  4.43it/s]
Iteration:   2%|▏         | 244/11591 [00:51<42:29,  4.45it/s]
Iteration:   2%|▏         | 245/11591 [00:52<42:46,  4.42it/s]
Iteration:   2%|▏         | 246/11591 [00:52<42:23,  4.46it/s]
Iteration:   2%|▏         | 247/11591 [00:52<42:02,  4.50it/s]
Iteration:   2%|▏         | 248/11591 [00:52<42:10,  4.48it/s]
Iteration:   2%|▏         | 249/11591 [00:53<41:46,  4.52it/s]
Iteration:   2%|▏         | 250/11591 [00:53<41:27,  4.56it/s]
Iteration:   2%|▏         | 251/11591 [00:53<41:35,  4.54it/s]
Iteration:   2%|▏         | 252/11591 [00:53<41:50,  4.52it/s]
Iteration:   2%|▏         | 253/11591 [00:53<41:43,  4.53it/s]
Iteration:   2%|▏         | 254/11591 [00:54<41:27,  4.56it/s]
Iteration:   2%|▏         | 255/11591 [00:54<41:39,  4.54it/s]
Iteration:   2%|▏         | 256/11591 [00:54<41:09,  4.59it/s]
Iteration:   2%|▏         | 257/11591 [00:54<40:56,  4.61it/s]
Iteration:   2%|▏         | 258/11591 [00:54<41:22,  4.57it/s]
Iteration:   2%|▏         | 259/11591 [00:55<41:11,  4.58it/s]
Iteration:   2%|▏         | 260/11591 [00:55<41:36,  4.54it/s]
Iteration:   2%|▏         | 261/11591 [00:55<41:56,  4.50it/s]
Iteration:   2%|▏         | 262/11591 [00:55<41:37,  4.54it/s]
Iteration:   2%|▏         | 263/11591 [00:56<41:11,  4.58it/s]
Iteration:   2%|▏         | 264/11591 [00:56<41:32,  4.54it/s]
Iteration:   2%|▏         | 265/11591 [00:56<41:21,  4.56it/s]
Iteration:   2%|▏         | 266/11591 [00:56<41:01,  4.60it/s]
Iteration:   2%|▏         | 267/11591 [00:56<41:17,  4.57it/s]
Iteration:   2%|▏         | 268/11591 [00:57<40:55,  4.61it/s]
Iteration:   2%|▏         | 269/11591 [00:57<40:29,  4.66it/s]
Iteration:   2%|▏         | 270/11591 [00:57<40:40,  4.64it/s]
Iteration:   2%|▏         | 271/11591 [00:57<40:37,  4.64it/s]
Iteration:   2%|▏         | 272/11591 [00:57<40:32,  4.65it/s]
Iteration:   2%|▏         | 273/11591 [00:58<41:03,  4.59it/s]
Iteration:   2%|▏         | 274/11591 [00:58<40:47,  4.62it/s]
Iteration:   2%|▏         | 275/11591 [00:58<40:19,  4.68it/s]
Iteration:   2%|▏         | 276/11591 [00:58<40:09,  4.70it/s]
Iteration:   2%|▏         | 277/11591 [00:59<40:21,  4.67it/s]
Iteration:   2%|▏         | 278/11591 [00:59<40:11,  4.69it/s]
Iteration:   2%|▏         | 279/11591 [00:59<39:41,  4.75it/s]
Iteration:   2%|▏         | 280/11591 [00:59<40:01,  4.71it/s]
Iteration:   2%|▏         | 281/11591 [00:59<39:50,  4.73it/s]
Iteration:   2%|▏         | 282/11591 [00:59<39:21,  4.79it/s]
Iteration:   2%|▏         | 283/11591 [01:00<39:43,  4.74it/s]
Iteration:   2%|▏         | 284/11591 [01:00<39:30,  4.77it/s]
Iteration:   2%|▏         | 285/11591 [01:00<39:08,  4.81it/s]
Iteration:   2%|▏         | 286/11591 [01:00<39:33,  4.76it/s]
Iteration:   2%|▏         | 287/11591 [01:01<39:19,  4.79it/s]
Iteration:   2%|▏         | 288/11591 [01:01<39:14,  4.80it/s]
Iteration:   2%|▏         | 289/11591 [01:01<39:35,  4.76it/s]
Iteration:   3%|▎         | 290/11591 [01:01<39:06,  4.82it/s]
Iteration:   3%|▎         | 291/11591 [01:01<38:37,  4.88it/s]
Iteration:   3%|▎         | 292/11591 [01:02<38:54,  4.84it/s]
Iteration:   3%|▎         | 293/11591 [01:02<38:44,  4.86it/s]
Iteration:   3%|▎         | 294/11591 [01:02<38:18,  4.92it/s]
Iteration:   3%|▎         | 295/11591 [01:02<38:15,  4.92it/s]
Iteration:   3%|▎         | 296/11591 [01:02<38:43,  4.86it/s]
Iteration:   3%|▎         | 297/11591 [01:02<38:27,  4.90it/s]
Iteration:   3%|▎         | 298/11591 [01:03<38:02,  4.95it/s]
Iteration:   3%|▎         | 299/11591 [01:03<38:49,  4.85it/s]
Iteration:   3%|▎         | 300/11591 [01:03<39:09,  4.81it/s]
Iteration:   3%|▎         | 301/11591 [01:03<39:07,  4.81it/s]
Iteration:   3%|▎         | 302/11591 [01:04<39:52,  4.72it/s]
Iteration:   3%|▎         | 303/11591 [01:04<40:04,  4.69it/s]
Iteration:   3%|▎         | 304/11591 [01:04<40:10,  4.68it/s]
Iteration:   3%|▎         | 305/11591 [01:04<41:16,  4.56it/s]
Iteration:   3%|▎         | 306/11591 [01:05<40:59,  4.59it/s]
Iteration:   3%|▎         | 307/11591 [01:05<40:44,  4.62it/s]
Iteration:   3%|▎         | 308/11591 [01:05<40:43,  4.62it/s]
Iteration:   3%|▎         | 309/11591 [01:05<40:13,  4.67it/s]
Iteration:   3%|▎         | 310/11591 [01:05<39:43,  4.73it/s]
Iteration:   3%|▎         | 311/11591 [01:06<39:54,  4.71it/s]
Iteration:   3%|▎         | 312/11591 [01:06<39:33,  4.75it/s]
Iteration:   3%|▎         | 313/11591 [01:06<39:15,  4.79it/s]
Iteration:   3%|▎         | 314/11591 [01:06<40:18,  4.66it/s]
Iteration:   3%|▎         | 315/11591 [01:06<39:52,  4.71it/s]
Iteration:   3%|▎         | 316/11591 [01:07<39:43,  4.73it/s]
Iteration:   3%|▎         | 317/11591 [01:07<39:35,  4.75it/s]
Iteration:   3%|▎         | 318/11591 [01:07<39:42,  4.73it/s]
Iteration:   3%|▎         | 319/11591 [01:07<39:39,  4.74it/s]
Iteration:   3%|▎         | 320/11591 [01:07<39:32,  4.75it/s]
Iteration:   3%|▎         | 321/11591 [01:08<40:00,  4.70it/s]
Iteration:   3%|▎         | 322/11591 [01:08<39:57,  4.70it/s]
Iteration:   3%|▎         | 323/11591 [01:08<39:53,  4.71it/s]
Iteration:   3%|▎         | 324/11591 [01:08<40:39,  4.62it/s]
Iteration:   3%|▎         | 325/11591 [01:09<40:28,  4.64it/s]
Iteration:   3%|▎         | 326/11591 [01:09<40:34,  4.63it/s]
Iteration:   3%|▎         | 327/11591 [01:09<40:37,  4.62it/s]
Iteration:   3%|▎         | 328/11591 [01:09<40:09,  4.67it/s]
Iteration:   3%|▎         | 329/11591 [01:09<39:51,  4.71it/s]
Iteration:   3%|▎         | 330/11591 [01:10<39:51,  4.71it/s]
Iteration:   3%|▎         | 331/11591 [01:10<39:35,  4.74it/s]
Iteration:   3%|▎         | 332/11591 [01:10<39:17,  4.78it/s]
Iteration:   3%|▎         | 333/11591 [01:10<39:48,  4.71it/s]
Iteration:   3%|▎         | 334/11591 [01:10<39:20,  4.77it/s]
Iteration:   3%|▎         | 335/11591 [01:11<39:15,  4.78it/s]
Iteration:   3%|▎         | 336/11591 [01:11<39:33,  4.74it/s]
Iteration:   3%|▎         | 337/11591 [01:11<39:13,  4.78it/s]
Iteration:   3%|▎         | 338/11591 [01:11<39:03,  4.80it/s]
Iteration:   3%|▎         | 339/11591 [01:11<38:56,  4.82it/s]
Iteration:   3%|▎         | 340/11591 [01:12<39:08,  4.79it/s]
Iteration:   3%|▎         | 341/11591 [01:12<39:06,  4.79it/s]
Iteration:   3%|▎         | 342/11591 [01:12<38:33,  4.86it/s]
Iteration:   3%|▎         | 343/11591 [01:12<38:55,  4.82it/s]
Iteration:   3%|▎         | 344/11591 [01:12<38:43,  4.84it/s]
Iteration:   3%|▎         | 345/11591 [01:13<38:27,  4.87it/s]
Iteration:   3%|▎         | 346/11591 [01:13<38:49,  4.83it/s]
Iteration:   3%|▎         | 347/11591 [01:13<38:38,  4.85it/s]
Iteration:   3%|▎         | 348/11591 [01:13<38:33,  4.86it/s]
Iteration:   3%|▎         | 349/11591 [01:14<38:55,  4.81it/s]
Iteration:   3%|▎         | 350/11591 [01:14<38:56,  4.81it/s]
Iteration:   3%|▎         | 351/11591 [01:14<38:38,  4.85it/s]
Iteration:   3%|▎         | 352/11591 [01:14<38:45,  4.83it/s]
Iteration:   3%|▎         | 353/11591 [01:14<38:30,  4.86it/s]
Iteration:   3%|▎         | 354/11591 [01:15<38:25,  4.87it/s]
Iteration:   3%|▎         | 355/11591 [01:15<38:56,  4.81it/s]
Iteration:   3%|▎         | 356/11591 [01:15<38:48,  4.82it/s]
Iteration:   3%|▎         | 357/11591 [01:15<38:38,  4.85it/s]
Iteration:   3%|▎         | 358/11591 [01:15<39:05,  4.79it/s]
Iteration:   3%|▎         | 359/11591 [01:16<39:09,  4.78it/s]
Iteration:   3%|▎         | 360/11591 [01:16<39:00,  4.80it/s]
Iteration:   3%|▎         | 361/11591 [01:16<38:44,  4.83it/s]
Iteration:   3%|▎         | 362/11591 [01:16<39:01,  4.80it/s]
Iteration:   3%|▎         | 363/11591 [01:16<38:48,  4.82it/s]
Iteration:   3%|▎         | 364/11591 [01:17<38:40,  4.84it/s]
Iteration:   3%|▎         | 365/11591 [01:17<38:57,  4.80it/s]
Iteration:   3%|▎         | 366/11591 [01:17<38:40,  4.84it/s]
Iteration:   3%|▎         | 367/11591 [01:17<38:28,  4.86it/s]
Iteration:   3%|▎         | 368/11591 [01:17<38:54,  4.81it/s]
Iteration:   3%|▎         | 369/11591 [01:18<38:48,  4.82it/s]
Iteration:   3%|▎         | 370/11591 [01:18<38:46,  4.82it/s]
Iteration:   3%|▎         | 371/11591 [01:18<39:18,  4.76it/s]
Iteration:   3%|▎         | 372/11591 [01:18<39:09,  4.77it/s]
Iteration:   3%|▎         | 373/11591 [01:19<38:55,  4.80it/s]
Iteration:   3%|▎         | 374/11591 [01:19<39:20,  4.75it/s]
Iteration:   3%|▎         | 375/11591 [01:19<39:10,  4.77it/s]
Iteration:   3%|▎         | 376/11591 [01:19<38:57,  4.80it/s]
Iteration:   3%|▎         | 377/11591 [01:19<39:24,  4.74it/s]
Iteration:   3%|▎         | 378/11591 [01:20<39:20,  4.75it/s]
Iteration:   3%|▎         | 379/11591 [01:20<39:05,  4.78it/s]
Iteration:   3%|▎         | 380/11591 [01:20<39:28,  4.73it/s]
Iteration:   3%|▎         | 381/11591 [01:20<39:07,  4.78it/s]
Iteration:   3%|▎         | 382/11591 [01:20<38:49,  4.81it/s]
Iteration:   3%|▎         | 383/11591 [01:21<38:23,  4.87it/s]
Iteration:   3%|▎         | 384/11591 [01:21<38:46,  4.82it/s]
Iteration:   3%|▎         | 385/11591 [01:21<38:34,  4.84it/s]
Iteration:   3%|▎         | 386/11591 [01:21<38:10,  4.89it/s]
Iteration:   3%|▎         | 387/11591 [01:21<38:36,  4.84it/s]
Iteration:   3%|▎         | 388/11591 [01:22<38:18,  4.87it/s]
Iteration:   3%|▎         | 389/11591 [01:22<38:13,  4.88it/s]
Iteration:   3%|▎         | 390/11591 [01:22<38:31,  4.85it/s]
Iteration:   3%|▎         | 391/11591 [01:22<38:19,  4.87it/s]
Iteration:   3%|▎         | 392/11591 [01:22<38:14,  4.88it/s]
Iteration:   3%|▎         | 393/11591 [01:23<38:46,  4.81it/s]
Iteration:   3%|▎         | 394/11591 [01:23<38:30,  4.85it/s]
Iteration:   3%|▎         | 395/11591 [01:23<38:12,  4.88it/s]
Iteration:   3%|▎         | 396/11591 [01:23<38:28,  4.85it/s]
Iteration:   3%|▎         | 397/11591 [01:23<38:20,  4.87it/s]
Iteration:   3%|▎         | 398/11591 [01:24<38:17,  4.87it/s]
Iteration:   3%|▎         | 399/11591 [01:24<38:45,  4.81it/s]
Iteration:   3%|▎         | 400/11591 [01:24<38:32,  4.84it/s]
Iteration:   3%|▎         | 401/11591 [01:24<38:30,  4.84it/s]
Iteration:   3%|▎         | 402/11591 [01:25<38:42,  4.82it/s]
Iteration:   3%|▎         | 403/11591 [01:25<38:28,  4.85it/s]
Iteration:   3%|▎         | 404/11591 [01:25<38:09,  4.89it/s]
Iteration:   3%|▎         | 405/11591 [01:25<38:03,  4.90it/s]
Iteration:   4%|▎         | 406/11591 [01:25<38:35,  4.83it/s]
Iteration:   4%|▎         | 407/11591 [01:26<38:34,  4.83it/s]
Iteration:   4%|▎         | 408/11591 [01:26<38:09,  4.88it/s]
Iteration:   4%|▎         | 409/11591 [01:26<38:27,  4.85it/s]
Iteration:   4%|▎         | 410/11591 [01:26<38:16,  4.87it/s]
Iteration:   4%|▎         | 411/11591 [01:26<38:04,  4.89it/s]
Iteration:   4%|▎         | 412/11591 [01:27<38:39,  4.82it/s]
Iteration:   4%|▎         | 413/11591 [01:27<38:38,  4.82it/s]
Iteration:   4%|▎         | 414/11591 [01:27<39:21,  4.73it/s]
Iteration:   4%|▎         | 415/11591 [01:27<40:42,  4.58it/s]
Iteration:   4%|▎         | 416/11591 [01:28<40:44,  4.57it/s]
Iteration:   4%|▎         | 417/11591 [01:28<40:18,  4.62it/s]
Iteration:   4%|▎         | 418/11591 [01:28<40:36,  4.59it/s]
Iteration:   4%|▎         | 419/11591 [01:28<40:18,  4.62it/s]
Iteration:   4%|▎         | 420/11591 [01:29<40:41,  4.57it/s]
Iteration:   4%|▎         | 421/11591 [01:29<41:24,  4.50it/s]
Iteration:   4%|▎         | 422/11591 [01:29<41:13,  4.51it/s]
Iteration:   4%|▎         | 423/11591 [01:29<41:15,  4.51it/s]
Iteration:   4%|▎         | 424/11591 [01:30<41:49,  4.45it/s]
Iteration:   4%|▎         | 425/11591 [01:30<41:49,  4.45it/s]
Iteration:   4%|▎         | 426/11591 [01:30<41:44,  4.46it/s]
Iteration:   4%|▎         | 427/11591 [01:30<41:26,  4.49it/s]
Iteration:   4%|▎         | 428/11591 [01:30<41:47,  4.45it/s]
Iteration:   4%|▎         | 429/11591 [01:31<41:17,  4.51it/s]
Iteration:   4%|▎         | 430/11591 [01:31<41:00,  4.54it/s]
Iteration:   4%|▎         | 431/11591 [01:31<40:56,  4.54it/s]
Iteration:   4%|▎         | 432/11591 [01:31<40:22,  4.61it/s]
Iteration:   4%|▎         | 433/11591 [01:31<40:05,  4.64it/s]
Iteration:   4%|▎         | 434/11591 [01:32<40:34,  4.58it/s]
Iteration:   4%|▍         | 435/11591 [01:32<40:19,  4.61it/s]
Iteration:   4%|▍         | 436/11591 [01:32<39:48,  4.67it/s]
Iteration:   4%|▍         | 437/11591 [01:32<39:33,  4.70it/s]
Iteration:   4%|▍         | 438/11591 [01:32<40:13,  4.62it/s]
Iteration:   4%|▍         | 439/11591 [01:33<39:54,  4.66it/s]
Iteration:   4%|▍         | 440/11591 [01:33<39:45,  4.67it/s]
Iteration:   4%|▍         | 441/11591 [01:33<39:57,  4.65it/s]
Iteration:   4%|▍         | 442/11591 [01:33<39:42,  4.68it/s]
Iteration:   4%|▍         | 443/11591 [01:33<39:30,  4.70it/s]
Iteration:   4%|▍         | 444/11591 [01:34<40:25,  4.60it/s]
Iteration:   4%|▍         | 445/11591 [01:34<40:27,  4.59it/s]
Iteration:   4%|▍         | 446/11591 [01:34<40:15,  4.61it/s]
Iteration:   4%|▍         | 447/11591 [01:34<40:50,  4.55it/s]
Iteration:   4%|▍         | 448/11591 [01:35<40:39,  4.57it/s]
Iteration:   4%|▍         | 449/11591 [01:35<40:19,  4.61it/s]
Iteration:   4%|▍         | 450/11591 [01:35<40:33,  4.58it/s]
Iteration:   4%|▍         | 451/11591 [01:35<40:16,  4.61it/s]
Iteration:   4%|▍         | 452/11591 [01:35<39:47,  4.67it/s]
Iteration:   4%|▍         | 453/11591 [01:36<39:28,  4.70it/s]
Iteration:   4%|▍         | 454/11591 [01:36<39:48,  4.66it/s]
Iteration:   4%|▍         | 455/11591 [01:36<39:43,  4.67it/s]
Iteration:   4%|▍         | 456/11591 [01:36<39:17,  4.72it/s]
Iteration:   4%|▍         | 457/11591 [01:36<39:32,  4.69it/s]
Iteration:   4%|▍         | 458/11591 [01:37<39:01,  4.76it/s]
Iteration:   4%|▍         | 459/11591 [01:37<38:47,  4.78it/s]
Iteration:   4%|▍         | 460/11591 [01:37<39:07,  4.74it/s]
Iteration:   4%|▍         | 461/11591 [01:37<38:43,  4.79it/s]
Iteration:   4%|▍         | 462/11591 [01:37<38:41,  4.79it/s]
Iteration:   4%|▍         | 463/11591 [01:38<39:00,  4.75it/s]
Iteration:   4%|▍         | 464/11591 [01:38<38:45,  4.79it/s]
Iteration:   4%|▍         | 465/11591 [01:38<38:25,  4.82it/s]
Iteration:   4%|▍         | 466/11591 [01:38<38:48,  4.78it/s]
Iteration:   4%|▍         | 467/11591 [01:38<38:32,  4.81it/s]
Iteration:   4%|▍         | 468/11591 [01:39<38:23,  4.83it/s]
Iteration:   4%|▍         | 469/11591 [01:39<39:00,  4.75it/s]
Iteration:   4%|▍         | 470/11591 [01:39<38:40,  4.79it/s]
Iteration:   4%|▍         | 471/11591 [01:39<38:19,  4.84it/s]
Iteration:   4%|▍         | 472/11591 [01:40<38:56,  4.76it/s]
Iteration:   4%|▍         | 473/11591 [01:40<38:50,  4.77it/s]
Iteration:   4%|▍         | 474/11591 [01:40<38:33,  4.81it/s]
Iteration:   4%|▍         | 475/11591 [01:40<38:22,  4.83it/s]
Iteration:   4%|▍         | 476/11591 [01:40<38:28,  4.82it/s]
Iteration:   4%|▍         | 477/11591 [01:40<38:11,  4.85it/s]
Iteration:   4%|▍         | 478/11591 [01:41<37:44,  4.91it/s]
Iteration:   4%|▍         | 479/11591 [01:41<38:14,  4.84it/s]
Iteration:   4%|▍         | 480/11591 [01:41<37:58,  4.88it/s]
Iteration:   4%|▍         | 481/11591 [01:41<37:53,  4.89it/s]
Iteration:   4%|▍         | 482/11591 [01:42<38:17,  4.83it/s]
Iteration:   4%|▍         | 483/11591 [01:42<37:55,  4.88it/s]
Iteration:   4%|▍         | 484/11591 [01:42<37:50,  4.89it/s]
Iteration:   4%|▍         | 485/11591 [01:42<38:39,  4.79it/s]
Iteration:   4%|▍         | 486/11591 [01:42<38:28,  4.81it/s]
Iteration:   4%|▍         | 487/11591 [01:43<38:39,  4.79it/s]
Iteration:   4%|▍         | 488/11591 [01:43<39:14,  4.72it/s]
Iteration:   4%|▍         | 489/11591 [01:43<39:11,  4.72it/s]
Iteration:   4%|▍         | 490/11591 [01:43<39:15,  4.71it/s]
Iteration:   4%|▍         | 491/11591 [01:44<40:11,  4.60it/s]
Iteration:   4%|▍         | 492/11591 [01:44<40:28,  4.57it/s]
Iteration:   4%|▍         | 493/11591 [01:44<40:32,  4.56it/s]
Iteration:   4%|▍         | 494/11591 [01:44<40:22,  4.58it/s]
Iteration:   4%|▍         | 495/11591 [01:45<40:49,  4.53it/s]
Iteration:   4%|▍         | 496/11591 [01:45<40:29,  4.57it/s]
Iteration:   4%|▍         | 497/11591 [01:45<40:39,  4.55it/s]
Iteration:   4%|▍         | 498/11591 [01:45<41:04,  4.50it/s]
Iteration:   4%|▍         | 499/11591 [01:45<41:00,  4.51it/s]
Iteration:   4%|▍         | 500/11591 [01:46<40:55,  4.52it/s]
Iteration:   4%|▍         | 501/11591 [01:46<41:25,  4.46it/s]
Iteration:   4%|▍         | 502/11591 [01:46<41:15,  4.48it/s]
Iteration:   4%|▍         | 503/11591 [01:46<41:03,  4.50it/s]
Iteration:   4%|▍         | 504/11591 [01:47<41:39,  4.44it/s]
Iteration:   4%|▍         | 505/11591 [01:47<41:21,  4.47it/s]
Iteration:   4%|▍         | 506/11591 [01:47<40:48,  4.53it/s]
Iteration:   4%|▍         | 507/11591 [01:47<41:09,  4.49it/s]
Iteration:   4%|▍         | 508/11591 [01:47<41:04,  4.50it/s]
Iteration:   4%|▍         | 509/11591 [01:48<40:42,  4.54it/s]
Iteration:   4%|▍         | 510/11591 [01:48<40:57,  4.51it/s]
Iteration:   4%|▍         | 511/11591 [01:48<40:37,  4.55it/s]
Iteration:   4%|▍         | 512/11591 [01:48<40:17,  4.58it/s]
Iteration:   4%|▍         | 513/11591 [01:48<40:03,  4.61it/s]
Iteration:   4%|▍         | 514/11591 [01:49<40:22,  4.57it/s]
Iteration:   4%|▍         | 515/11591 [01:49<40:04,  4.61it/s]
Iteration:   4%|▍         | 516/11591 [01:49<39:35,  4.66it/s]
Iteration:   4%|▍         | 517/11591 [01:49<39:58,  4.62it/s]
Iteration:   4%|▍         | 518/11591 [01:49<39:25,  4.68it/s]
Iteration:   4%|▍         | 519/11591 [01:50<39:15,  4.70it/s]
Iteration:   4%|▍         | 520/11591 [01:50<39:33,  4.66it/s]
Iteration:   4%|▍         | 521/11591 [01:50<39:26,  4.68it/s]
Iteration:   5%|▍         | 522/11591 [01:50<38:53,  4.74it/s]
Iteration:   5%|▍         | 523/11591 [01:51<39:07,  4.71it/s]
Iteration:   5%|▍         | 524/11591 [01:51<38:49,  4.75it/s]
Iteration:   5%|▍         | 525/11591 [01:51<38:55,  4.74it/s]
Iteration:   5%|▍         | 526/11591 [01:51<39:21,  4.69it/s]
Iteration:   5%|▍         | 527/11591 [01:51<39:09,  4.71it/s]
Iteration:   5%|▍         | 528/11591 [01:52<39:02,  4.72it/s]
Iteration:   5%|▍         | 529/11591 [01:52<39:18,  4.69it/s]
Iteration:   5%|▍         | 530/11591 [01:52<39:05,  4.72it/s]
Iteration:   5%|▍         | 531/11591 [01:52<39:03,  4.72it/s]
Iteration:   5%|▍         | 532/11591 [01:52<39:28,  4.67it/s]
Iteration:   5%|▍         | 533/11591 [01:53<39:18,  4.69it/s]
Iteration:   5%|▍         | 534/11591 [01:53<38:56,  4.73it/s]
Iteration:   5%|▍         | 535/11591 [01:53<39:03,  4.72it/s]
Iteration:   5%|▍         | 536/11591 [01:53<38:46,  4.75it/s]
Iteration:   5%|▍         | 537/11591 [01:53<38:41,  4.76it/s]
Iteration:   5%|▍         | 538/11591 [01:54<38:19,  4.81it/s]
Iteration:   5%|▍         | 539/11591 [01:54<38:43,  4.76it/s]
Iteration:   5%|▍         | 540/11591 [01:54<38:22,  4.80it/s]
Iteration:   5%|▍         | 541/11591 [01:54<38:17,  4.81it/s]
Iteration:   5%|▍         | 542/11591 [01:54<38:30,  4.78it/s]
Iteration:   5%|▍         | 543/11591 [01:55<38:12,  4.82it/s]
Iteration:   5%|▍         | 544/11591 [01:55<37:53,  4.86it/s]
Iteration:   5%|▍         | 545/11591 [01:55<38:15,  4.81it/s]
Iteration:   5%|▍         | 546/11591 [01:55<37:46,  4.87it/s]
Iteration:   5%|▍         | 547/11591 [01:55<37:30,  4.91it/s]
Iteration:   5%|▍         | 548/11591 [01:56<37:55,  4.85it/s]
Iteration:   5%|▍         | 549/11591 [01:56<37:42,  4.88it/s]
Iteration:   5%|▍         | 550/11591 [01:56<37:22,  4.92it/s]
Iteration:   5%|▍         | 551/11591 [01:56<37:35,  4.89it/s]
Iteration:   5%|▍         | 552/11591 [01:56<37:19,  4.93it/s]
Iteration:   5%|▍         | 553/11591 [01:57<37:05,  4.96it/s]
Iteration:   5%|▍         | 554/11591 [01:57<36:57,  4.98it/s]
Iteration:   5%|▍         | 555/11591 [01:57<37:30,  4.90it/s]
Iteration:   5%|▍         | 556/11591 [01:57<37:20,  4.92it/s]
Iteration:   5%|▍         | 557/11591 [01:57<37:17,  4.93it/s]
Iteration:   5%|▍         | 558/11591 [01:58<37:35,  4.89it/s]
Iteration:   5%|▍         | 559/11591 [01:58<37:11,  4.94it/s]
Iteration:   5%|▍         | 560/11591 [01:58<36:52,  4.99it/s]
Iteration:   5%|▍         | 561/11591 [01:58<37:23,  4.92it/s]
Iteration:   5%|▍         | 562/11591 [01:58<37:20,  4.92it/s]
Iteration:   5%|▍         | 563/11591 [01:59<37:10,  4.95it/s]
Iteration:   5%|▍         | 564/11591 [01:59<37:25,  4.91it/s]
Iteration:   5%|▍         | 565/11591 [01:59<37:14,  4.93it/s]
Iteration:   5%|▍         | 566/11591 [01:59<37:10,  4.94it/s]
Iteration:   5%|▍         | 567/11591 [01:59<37:37,  4.88it/s]
Iteration:   5%|▍         | 568/11591 [02:00<37:44,  4.87it/s]
Iteration:   5%|▍         | 569/11591 [02:00<37:52,  4.85it/s]
Iteration:   5%|▍         | 570/11591 [02:00<38:43,  4.74it/s]
Iteration:   5%|▍         | 571/11591 [02:00<38:37,  4.76it/s]
Iteration:   5%|▍         | 572/11591 [02:01<38:33,  4.76it/s]
Iteration:   5%|▍         | 573/11591 [02:01<38:51,  4.73it/s]
Iteration:   5%|▍         | 574/11591 [02:01<38:19,  4.79it/s]
Iteration:   5%|▍         | 575/11591 [02:01<38:08,  4.81it/s]
Iteration:   5%|▍         | 576/11591 [02:01<38:05,  4.82it/s]
Iteration:   5%|▍         | 577/11591 [02:02<38:20,  4.79it/s]
Iteration:   5%|▍         | 578/11591 [02:02<38:17,  4.79it/s]
Iteration:   5%|▍         | 579/11591 [02:02<38:28,  4.77it/s]
Iteration:   5%|▌         | 580/11591 [02:02<38:40,  4.74it/s]
Iteration:   5%|▌         | 581/11591 [02:02<38:15,  4.80it/s]
Iteration:   5%|▌         | 582/11591 [02:03<37:52,  4.84it/s]
Iteration:   5%|▌         | 583/11591 [02:03<38:11,  4.80it/s]
Iteration:   5%|▌         | 584/11591 [02:03<38:14,  4.80it/s]
Iteration:   5%|▌         | 585/11591 [02:03<37:57,  4.83it/s]
Iteration:   5%|▌         | 586/11591 [02:04<38:46,  4.73it/s]
Iteration:   5%|▌         | 587/11591 [02:04<38:58,  4.71it/s]
Iteration:   5%|▌         | 588/11591 [02:04<39:20,  4.66it/s]
Iteration:   5%|▌         | 589/11591 [02:04<39:56,  4.59it/s]
Iteration:   5%|▌         | 590/11591 [02:05<39:53,  4.60it/s]
Iteration:   5%|▌         | 591/11591 [02:05<40:01,  4.58it/s]
Iteration:   5%|▌         | 592/11591 [02:05<40:40,  4.51it/s]
Iteration:   5%|▌         | 593/11591 [02:05<41:11,  4.45it/s]
Iteration:   5%|▌         | 594/11591 [02:06<41:08,  4.45it/s]
Iteration:   5%|▌         | 595/11591 [02:06<41:42,  4.39it/s]
Iteration:   5%|▌         | 596/11591 [02:06<41:27,  4.42it/s]
Iteration:   5%|▌         | 597/11591 [02:06<41:54,  4.37it/s]
Iteration:   5%|▌         | 598/11591 [02:07<41:56,  4.37it/s]
Iteration:   5%|▌         | 599/11591 [02:07<42:39,  4.30it/s]
Iteration:   5%|▌         | 600/11591 [02:07<42:37,  4.30it/s]
Iteration:   5%|▌         | 601/11591 [02:07<42:30,  4.31it/s]
Iteration:   5%|▌         | 602/11591 [02:08<42:56,  4.26it/s]
Iteration:   5%|▌         | 603/11591 [02:08<42:35,  4.30it/s]
Iteration:   5%|▌         | 604/11591 [02:08<42:38,  4.29it/s]
Iteration:   5%|▌         | 605/11591 [02:08<43:24,  4.22it/s]
Iteration:   5%|▌         | 606/11591 [02:09<43:24,  4.22it/s]
Iteration:   5%|▌         | 607/11591 [02:09<44:19,  4.13it/s]
Iteration:   5%|▌         | 608/11591 [02:09<45:13,  4.05it/s]
Iteration:   5%|▌         | 609/11591 [02:10<45:21,  4.04it/s]
Iteration:   5%|▌         | 610/11591 [02:10<45:21,  4.04it/s]
Iteration:   5%|▌         | 611/11591 [02:10<46:09,  3.96it/s]
Iteration:   5%|▌         | 612/11591 [02:10<45:25,  4.03it/s]
Iteration:   5%|▌         | 613/11591 [02:10<44:38,  4.10it/s]
Iteration:   5%|▌         | 614/11591 [02:11<44:56,  4.07it/s]
Iteration:   5%|▌         | 615/11591 [02:11<44:44,  4.09it/s]
Iteration:   5%|▌         | 616/11591 [02:11<44:47,  4.08it/s]
Iteration:   5%|▌         | 617/11591 [02:12<45:06,  4.06it/s]
Iteration:   5%|▌         | 618/11591 [02:12<44:39,  4.10it/s]
Iteration:   5%|▌         | 619/11591 [02:12<44:10,  4.14it/s]
Iteration:   5%|▌         | 620/11591 [02:12<43:45,  4.18it/s]
Iteration:   5%|▌         | 621/11591 [02:12<43:56,  4.16it/s]
Iteration:   5%|▌         | 622/11591 [02:13<43:26,  4.21it/s]
Iteration:   5%|▌         | 623/11591 [02:13<42:58,  4.25it/s]
Iteration:   5%|▌         | 624/11591 [02:13<43:05,  4.24it/s]
Iteration:   5%|▌         | 625/11591 [02:13<42:41,  4.28it/s]
Iteration:   5%|▌         | 626/11591 [02:13<42:39,  4.28it/s]
Iteration:   5%|▌         | 627/11591 [02:14<43:03,  4.24it/s]
Iteration:   5%|▌         | 628/11591 [02:14<42:51,  4.26it/s]
Iteration:   5%|▌         | 629/11591 [02:14<42:11,  4.33it/s]
Iteration:   5%|▌         | 630/11591 [02:14<42:35,  4.29it/s]
Iteration:   5%|▌         | 631/11591 [02:15<42:15,  4.32it/s]
Iteration:   5%|▌         | 632/11591 [02:15<41:53,  4.36it/s]
Iteration:   5%|▌         | 633/11591 [02:15<41:54,  4.36it/s]
Iteration:   5%|▌         | 634/11591 [02:15<41:24,  4.41it/s]
Iteration:   5%|▌         | 635/11591 [02:15<41:10,  4.43it/s]
Iteration:   5%|▌         | 636/11591 [02:16<41:27,  4.40it/s]
Iteration:   5%|▌         | 637/11591 [02:16<41:14,  4.43it/s]
Iteration:   6%|▌         | 638/11591 [02:16<40:51,  4.47it/s]
Iteration:   6%|▌         | 639/11591 [02:16<41:11,  4.43it/s]
Iteration:   6%|▌         | 640/11591 [02:16<40:50,  4.47it/s]
Iteration:   6%|▌         | 641/11591 [02:17<40:39,  4.49it/s]
Iteration:   6%|▌         | 642/11591 [02:17<40:55,  4.46it/s]
Iteration:   6%|▌         | 643/11591 [02:17<41:02,  4.45it/s]
Iteration:   6%|▌         | 644/11591 [02:17<40:52,  4.46it/s]
Iteration:   6%|▌         | 645/11591 [02:18<40:31,  4.50it/s]
Iteration:   6%|▌         | 646/11591 [02:18<40:34,  4.49it/s]
Iteration:   6%|▌         | 647/11591 [02:18<40:28,  4.51it/s]
Iteration:   6%|▌         | 648/11591 [02:18<40:10,  4.54it/s]
Iteration:   6%|▌         | 649/11591 [02:18<40:21,  4.52it/s]
Iteration:   6%|▌         | 650/11591 [02:19<39:54,  4.57it/s]
Iteration:   6%|▌         | 651/11591 [02:19<39:41,  4.59it/s]
Iteration:   6%|▌         | 652/11591 [02:19<39:48,  4.58it/s]
Iteration:   6%|▌         | 653/11591 [02:19<39:16,  4.64it/s]
Iteration:   6%|▌         | 654/11591 [02:19<39:02,  4.67it/s]
Iteration:   6%|▌         | 655/11591 [02:20<39:05,  4.66it/s]
Iteration:   6%|▌         | 656/11591 [02:20<38:46,  4.70it/s]
Iteration:   6%|▌         | 657/11591 [02:20<38:52,  4.69it/s]
Iteration:   6%|▌         | 658/11591 [02:20<39:43,  4.59it/s]
Iteration:   6%|▌         | 659/11591 [02:20<39:34,  4.60it/s]
Iteration:   6%|▌         | 660/11591 [02:21<39:20,  4.63it/s]
Iteration:   6%|▌         | 661/11591 [02:21<39:30,  4.61it/s]
Iteration:   6%|▌         | 662/11591 [02:21<39:22,  4.63it/s]
Iteration:   6%|▌         | 663/11591 [02:21<39:21,  4.63it/s]
Iteration:   6%|▌         | 664/11591 [02:21<39:00,  4.67it/s]
Iteration:   6%|▌         | 665/11591 [02:22<39:14,  4.64it/s]
Iteration:   6%|▌         | 666/11591 [02:22<38:56,  4.68it/s]
Iteration:   6%|▌         | 667/11591 [02:22<38:35,  4.72it/s]
Iteration:   6%|▌         | 668/11591 [02:22<38:33,  4.72it/s]
Iteration:   6%|▌         | 669/11591 [02:22<38:16,  4.76it/s]
Iteration:   6%|▌         | 670/11591 [02:23<37:53,  4.80it/s]
Iteration:   6%|▌         | 671/11591 [02:23<38:09,  4.77it/s]
Iteration:   6%|▌         | 672/11591 [02:23<38:01,  4.79it/s]
Iteration:   6%|▌         | 673/11591 [02:23<37:52,  4.80it/s]
Iteration:   6%|▌         | 674/11591 [02:24<38:19,  4.75it/s]
Iteration:   6%|▌         | 675/11591 [02:24<38:15,  4.76it/s]
Iteration:   6%|▌         | 676/11591 [02:24<38:12,  4.76it/s]
Iteration:   6%|▌         | 677/11591 [02:24<38:33,  4.72it/s]
Iteration:   6%|▌         | 678/11591 [02:24<38:26,  4.73it/s]
Iteration:   6%|▌         | 679/11591 [02:25<38:28,  4.73it/s]
Iteration:   6%|▌         | 680/11591 [02:25<38:55,  4.67it/s]
Iteration:   6%|▌         | 681/11591 [02:25<38:52,  4.68it/s]
Iteration:   6%|▌         | 682/11591 [02:25<38:46,  4.69it/s]
Iteration:   6%|▌         | 683/11591 [02:26<39:08,  4.64it/s]
Iteration:   6%|▌         | 684/11591 [02:26<38:51,  4.68it/s]
Iteration:   6%|▌         | 685/11591 [02:26<38:43,  4.69it/s]
Iteration:   6%|▌         | 686/11591 [02:26<38:35,  4.71it/s]
Iteration:   6%|▌         | 687/11591 [02:26<39:11,  4.64it/s]
Iteration:   6%|▌         | 688/11591 [02:27<39:20,  4.62it/s]
Iteration:   6%|▌         | 689/11591 [02:27<39:56,  4.55it/s]
Iteration:   6%|▌         | 690/11591 [02:27<41:18,  4.40it/s]
Iteration:   6%|▌         | 691/11591 [02:28<41:36,  4.37it/s]
Iteration:   6%|▌         | 692/11591 [02:28<41:20,  4.39it/s]
Iteration:   6%|▌         | 693/11591 [02:28<42:09,  4.31it/s]
Iteration:   6%|▌         | 694/11591 [02:28<42:18,  4.29it/s]
Iteration:   6%|▌         | 695/11591 [02:29<42:23,  4.28it/s]
Iteration:   6%|▌         | 696/11591 [02:30<55:41,  3.26it/s]
Iteration:   6%|▌         | 697/11591 [02:33<1:15:13,  2.41it/s]
Iteration:   6%|▌         | 698/11591 [02:37<1:51:50,  1.62it/s]
Iteration:   6%|▌         | 699/11591 [02:37<1:48:34,  1.67it/s]
Iteration:   6%|▌         | 700/11591 [02:38<1:44:49,  1.73it/s]
Iteration:   6%|▌         | 701/11591 [02:38<1:41:28,  1.79it/s]
Iteration:   6%|▌         | 702/11591 [02:38<1:38:47,  1.84it/s]
Iteration:   6%|▌         | 703/11591 [02:38<1:35:48,  1.89it/s]
Iteration:   6%|▌         | 704/11591 [02:38<1:32:52,  1.95it/s]
Iteration:   6%|▌         | 705/11591 [02:39<1:30:45,  2.00it/s]
Iteration:   6%|▌         | 706/11591 [02:39<1:27:53,  2.06it/s]
Iteration:   6%|▌         | 707/11591 [02:39<1:25:04,  2.13it/s]
Iteration:   6%|▌         | 708/11591 [02:39<1:22:46,  2.19it/s]
Iteration:   6%|▌         | 709/11591 [02:40<1:20:53,  2.24it/s]
Iteration:   6%|▌         | 710/11591 [02:40<1:18:27,  2.31it/s]
Iteration:   6%|▌         | 711/11591 [02:40<1:16:20,  2.38it/s]
Iteration:   6%|▌         | 712/11591 [02:40<1:14:47,  2.42it/s]
Iteration:   6%|▌         | 713/11591 [02:40<1:12:51,  2.49it/s]
Iteration:   6%|▌         | 714/11591 [02:41<1:11:12,  2.55it/s]
Iteration:   6%|▌         | 715/11591 [02:41<1:10:12,  2.58it/s]
Iteration:   6%|▌         | 716/11591 [02:41<1:08:33,  2.64it/s]
Iteration:   6%|▌         | 717/11591 [02:41<1:07:15,  2.69it/s]
Iteration:   6%|▌         | 718/11591 [02:42<1:06:19,  2.73it/s]
Iteration:   6%|▌         | 719/11591 [02:42<1:04:50,  2.79it/s]
Iteration:   6%|▌         | 720/11591 [02:42<1:03:27,  2.85it/s]
Iteration:   6%|▌         | 721/11591 [02:42<1:02:52,  2.88it/s]
Iteration:   6%|▌         | 722/11591 [02:44<1:13:47,  2.45it/s]
Iteration:   6%|▌         | 723/11591 [02:48<1:50:47,  1.63it/s]
Iteration:   6%|▌         | 724/11591 [02:49<1:50:34,  1.64it/s]
Iteration:   6%|▋         | 725/11591 [02:49<1:47:59,  1.68it/s]
Iteration:   6%|▋         | 726/11591 [02:50<1:45:42,  1.71it/s]
Iteration:   6%|▋         | 727/11591 [02:50<1:44:02,  1.74it/s]
Iteration:   6%|▋         | 728/11591 [02:50<1:40:45,  1.80it/s]
Iteration:   6%|▋         | 729/11591 [02:50<1:37:19,  1.86it/s]
Iteration:   6%|▋         | 730/11591 [02:51<1:35:12,  1.90it/s]
Iteration:   6%|▋         | 731/11591 [02:53<1:48:24,  1.67it/s]
Iteration:   6%|▋         | 732/11591 [02:53<1:45:48,  1.71it/s]
Iteration:   6%|▋         | 733/11591 [02:53<1:43:24,  1.75it/s]
Iteration:   6%|▋         | 734/11591 [02:54<1:41:31,  1.78it/s]
Iteration:   6%|▋         | 735/11591 [02:54<1:38:58,  1.83it/s]
Iteration:   6%|▋         | 736/11591 [02:54<1:37:11,  1.86it/s]
Iteration:   6%|▋         | 737/11591 [02:55<1:35:21,  1.90it/s]
Iteration:   6%|▋         | 738/11591 [02:55<1:32:33,  1.95it/s]
Iteration:   6%|▋         | 739/11591 [02:55<1:30:30,  2.00it/s]
Iteration:   6%|▋         | 740/11591 [02:56<1:29:04,  2.03it/s]
Iteration:   6%|▋         | 741/11591 [02:56<1:26:54,  2.08it/s]
Iteration:   6%|▋         | 742/11591 [02:56<1:24:44,  2.13it/s]
Iteration:   6%|▋         | 743/11591 [02:56<1:23:08,  2.17it/s]
Iteration:   6%|▋         | 744/11591 [02:56<1:20:28,  2.25it/s]
Iteration:   6%|▋         | 745/11591 [02:57<1:18:24,  2.31it/s]
Iteration:   6%|▋         | 746/11591 [02:57<1:18:55,  2.29it/s]
Iteration:   6%|▋         | 747/11591 [02:57<1:17:37,  2.33it/s]
Iteration:   6%|▋         | 748/11591 [02:58<1:15:36,  2.39it/s]
Iteration:   6%|▋         | 749/11591 [02:58<1:17:09,  2.34it/s]
Iteration:   6%|▋         | 750/11591 [02:59<1:16:46,  2.35it/s]
Iteration:   6%|▋         | 751/11591 [02:59<1:17:52,  2.32it/s]
Iteration:   6%|▋         | 752/11591 [03:01<1:32:01,  1.96it/s]
Iteration:   6%|▋         | 753/11591 [03:01<1:29:08,  2.03it/s]
Iteration:   7%|▋         | 754/11591 [03:02<1:27:27,  2.07it/s]
Iteration:   7%|▋         | 755/11591 [03:03<1:34:17,  1.92it/s]
Iteration:   7%|▋         | 756/11591 [03:03<1:32:02,  1.96it/s]
Iteration:   7%|▋         | 757/11591 [03:05<1:46:46,  1.69it/s]
Iteration:   7%|▋         | 758/11591 [03:06<1:42:59,  1.75it/s]
Iteration:   7%|▋         | 759/11591 [03:06<1:40:11,  1.80it/s]
Iteration:   7%|▋         | 760/11591 [03:07<1:45:24,  1.71it/s]
Iteration:   7%|▋         | 761/11591 [03:07<1:42:47,  1.76it/s]
Iteration:   7%|▋         | 762/11591 [03:09<1:55:34,  1.56it/s]
Iteration:   7%|▋         | 763/11591 [03:09<1:51:27,  1.62it/s]
Iteration:   7%|▋         | 764/11591 [03:12<2:13:04,  1.36it/s]
Iteration:   7%|▋         | 765/11591 [03:13<2:08:47,  1.40it/s]
Iteration:   7%|▋         | 766/11591 [03:13<2:05:33,  1.44it/s]
Iteration:   7%|▋         | 767/11591 [03:15<2:13:40,  1.35it/s]
Iteration:   7%|▋         | 768/11591 [03:15<2:09:03,  1.40it/s]
Iteration:   7%|▋         | 769/11591 [03:15<2:04:11,  1.45it/s]
Iteration:   7%|▋         | 770/11591 [03:15<2:02:00,  1.48it/s]
Iteration:   7%|▋         | 771/11591 [03:17<2:06:10,  1.43it/s]
Iteration:   7%|▋         | 772/11591 [03:18<2:10:03,  1.39it/s]
Iteration:   7%|▋         | 773/11591 [03:18<2:10:01,  1.39it/s]
Iteration:   7%|▋         | 774/11591 [03:20<2:20:11,  1.29it/s]
Iteration:   7%|▋         | 775/11591 [03:21<2:17:00,  1.32it/s]
Iteration:   7%|▋         | 776/11591 [03:21<2:13:31,  1.35it/s]
Iteration:   7%|▋         | 777/11591 [03:22<2:11:48,  1.37it/s]
Iteration:   7%|▋         | 778/11591 [03:23<2:18:39,  1.30it/s]
Iteration:   7%|▋         | 779/11591 [03:23<2:13:25,  1.35it/s]
Iteration:   7%|▋         | 780/11591 [03:24<2:08:24,  1.40it/s]
Iteration:   7%|▋         | 781/11591 [03:25<2:16:55,  1.32it/s]
Iteration:   7%|▋         | 782/11591 [03:25<2:11:29,  1.37it/s]
Iteration:   7%|▋         | 783/11591 [03:26<2:06:42,  1.42it/s]
Iteration:   7%|▋         | 784/11591 [03:28<2:24:58,  1.24it/s]
Iteration:   7%|▋         | 785/11591 [03:41<4:13:14,  1.41s/it]
Iteration:   7%|▋         | 786/11591 [03:41<4:03:13,  1.35s/it]
Iteration:   7%|▋         | 787/11591 [03:55<5:51:22,  1.95s/it]
Iteration:   7%|▋         | 788/11591 [03:55<5:35:33,  1.86s/it]
Iteration:   7%|▋         | 789/11591 [03:55<5:20:51,  1.78s/it]
Iteration:   7%|▋         | 790/11591 [03:56<5:07:35,  1.71s/it]
Iteration:   7%|▋         | 791/11591 [03:56<4:54:23,  1.64s/it]
Iteration:   7%|▋         | 792/11591 [03:56<4:41:41,  1.57s/it]
Iteration:   7%|▋         | 793/11591 [03:56<4:30:05,  1.50s/it]
Iteration:   7%|▋         | 794/11591 [03:57<4:18:38,  1.44s/it]
Iteration:   7%|▋         | 795/11591 [03:57<4:08:03,  1.38s/it]
Iteration:   7%|▋         | 796/11591 [03:57<3:57:47,  1.32s/it]
Iteration:   7%|▋         | 797/11591 [03:57<3:48:16,  1.27s/it]
Iteration:   7%|▋         | 798/11591 [03:58<3:38:54,  1.22s/it]
Iteration:   7%|▋         | 799/11591 [03:58<3:30:01,  1.17s/it]
Iteration:   7%|▋         | 800/11591 [03:58<3:22:57,  1.13s/it]
Iteration:   7%|▋         | 801/11591 [03:58<3:14:55,  1.08s/it]
Iteration:   7%|▋         | 802/11591 [03:59<3:07:15,  1.04s/it]
Iteration:   7%|▋         | 803/11591 [03:59<3:00:24,  1.00s/it]
Iteration:   7%|▋         | 804/11591 [03:59<2:53:17,  1.04it/s]
Iteration:   7%|▋         | 805/11591 [04:00<2:52:48,  1.04it/s]
Iteration:   7%|▋         | 806/11591 [04:03<3:13:18,  1.08s/it]
Iteration:   7%|▋         | 807/11591 [04:06<3:31:55,  1.18s/it]
Iteration:   7%|▋         | 808/11591 [04:07<3:28:35,  1.16s/it]
Iteration:   7%|▋         | 809/11591 [04:09<3:36:25,  1.20s/it]
Iteration:   7%|▋         | 810/11591 [04:18<4:48:06,  1.60s/it]
Iteration:   7%|▋         | 811/11591 [04:19<4:39:07,  1.55s/it]
Iteration:   7%|▋         | 812/11591 [04:20<4:33:34,  1.52s/it]
Iteration:   7%|▋         | 813/11591 [04:21<4:24:53,  1.47s/it]
Iteration:   7%|▋         | 814/11591 [04:22<4:22:58,  1.46s/it]
Iteration:   7%|▋         | 815/11591 [04:23<4:24:18,  1.47s/it]
Iteration:   7%|▋         | 816/11591 [04:24<4:19:23,  1.44s/it]
Iteration:   7%|▋         | 817/11591 [04:26<4:21:40,  1.46s/it]
Iteration:   7%|▋         | 818/11591 [04:27<4:18:44,  1.44s/it]
Iteration:   7%|▋         | 819/11591 [04:28<4:11:44,  1.40s/it]
Iteration:   7%|▋         | 820/11591 [04:28<4:03:49,  1.36s/it]
Iteration:   7%|▋         | 821/11591 [04:29<3:56:35,  1.32s/it]
Iteration:   7%|▋         | 822/11591 [04:30<3:50:32,  1.28s/it]
Iteration:   7%|▋         | 823/11591 [04:30<3:45:24,  1.26s/it]
Iteration:   7%|▋         | 824/11591 [04:31<3:39:01,  1.22s/it]
Iteration:   7%|▋         | 825/11591 [04:32<3:34:47,  1.20s/it]
Iteration:   7%|▋         | 826/11591 [04:32<3:29:01,  1.16s/it]
Iteration:   7%|▋         | 827/11591 [04:33<3:25:47,  1.15s/it]
Iteration:   7%|▋         | 828/11591 [04:34<3:23:23,  1.13s/it]
Iteration:   7%|▋         | 829/11591 [04:34<3:18:01,  1.10s/it]
Iteration:   7%|▋         | 830/11591 [04:35<3:14:38,  1.09s/it]
Iteration:   7%|▋         | 831/11591 [04:36<3:13:19,  1.08s/it]
Iteration:   7%|▋         | 832/11591 [04:37<3:08:25,  1.05s/it]
Iteration:   7%|▋         | 833/11591 [04:37<3:03:10,  1.02s/it]
Iteration:   7%|▋         | 834/11591 [04:38<3:01:04,  1.01s/it]
Iteration:   7%|▋         | 835/11591 [04:38<2:56:29,  1.02it/s]
Iteration:   7%|▋         | 836/11591 [04:39<2:53:17,  1.03it/s]
Iteration:   7%|▋         | 837/11591 [04:40<2:55:38,  1.02it/s]
Iteration:   7%|▋         | 838/11591 [04:41<2:56:00,  1.02it/s]
Iteration:   7%|▋         | 839/11591 [04:42<2:55:42,  1.02it/s]
Iteration:   7%|▋         | 840/11591 [04:43<2:54:43,  1.03it/s]
Iteration:   7%|▋         | 841/11591 [04:44<2:54:20,  1.03it/s]
Iteration:   7%|▋         | 842/11591 [04:45<2:52:23,  1.04it/s]
Iteration:   7%|▋         | 843/11591 [04:46<2:51:20,  1.05it/s]
Iteration:   7%|▋         | 844/11591 [04:46<2:50:24,  1.05it/s]
Iteration:   7%|▋         | 845/11591 [04:47<2:47:48,  1.07it/s]
Iteration:   7%|▋         | 846/11591 [04:48<2:45:36,  1.08it/s]
Iteration:   7%|▋         | 847/11591 [04:49<2:45:05,  1.08it/s]
Iteration:   7%|▋         | 848/11591 [04:49<2:44:07,  1.09it/s]
Iteration:   7%|▋         | 849/11591 [04:53<3:04:48,  1.03s/it]
Iteration:   7%|▋         | 850/11591 [04:55<3:12:05,  1.07s/it]
Iteration:   7%|▋         | 851/11591 [04:55<3:07:36,  1.05s/it]
Iteration:   7%|▋         | 852/11591 [04:56<3:03:11,  1.02s/it]
Iteration:   7%|▋         | 853/11591 [04:57<3:02:53,  1.02s/it]
Iteration:   7%|▋         | 854/11591 [04:58<3:03:10,  1.02s/it]
Iteration:   7%|▋         | 855/11591 [04:59<3:08:49,  1.06s/it]
Iteration:   7%|▋         | 856/11591 [05:01<3:12:57,  1.08s/it]
Iteration:   7%|▋         | 857/11591 [05:02<3:14:22,  1.09s/it]
Iteration:   7%|▋         | 858/11591 [05:04<3:20:45,  1.12s/it]
Iteration:   7%|▋         | 859/11591 [05:05<3:22:24,  1.13s/it]
Iteration:   7%|▋         | 860/11591 [05:06<3:20:38,  1.12s/it]
Iteration:   7%|▋         | 861/11591 [05:06<3:12:15,  1.08s/it]
Iteration:   7%|▋         | 862/11591 [05:07<3:04:06,  1.03s/it]
Iteration:   7%|▋         | 863/11591 [05:08<3:03:52,  1.03s/it]
Iteration:   7%|▋         | 864/11591 [05:08<3:02:06,  1.02s/it]
Iteration:   7%|▋         | 865/11591 [05:09<3:02:48,  1.02s/it]
Iteration:   7%|▋         | 866/11591 [05:10<3:02:29,  1.02s/it]
Iteration:   7%|▋         | 867/11591 [05:11<3:01:54,  1.02s/it]
Iteration:   7%|▋         | 868/11591 [05:12<2:59:01,  1.00s/it]
Iteration:   7%|▋         | 869/11591 [05:13<2:59:33,  1.00s/it]
Iteration:   8%|▊         | 870/11591 [05:14<2:57:07,  1.01it/s]
Iteration:   8%|▊         | 871/11591 [05:15<2:58:27,  1.00it/s]
Iteration:   8%|▊         | 872/11591 [05:15<2:52:16,  1.04it/s]
Iteration:   8%|▊         | 873/11591 [05:16<2:45:14,  1.08it/s]
Iteration:   8%|▊         | 874/11591 [05:16<2:38:51,  1.12it/s]
Iteration:   8%|▊         | 875/11591 [05:16<2:33:33,  1.16it/s]
Iteration:   8%|▊         | 876/11591 [05:16<2:27:38,  1.21it/s]
Iteration:   8%|▊         | 877/11591 [05:16<2:22:10,  1.26it/s]
Iteration:   8%|▊         | 878/11591 [05:17<2:19:11,  1.28it/s]
Iteration:   8%|▊         | 879/11591 [05:17<2:14:24,  1.33it/s]
Iteration:   8%|▊         | 880/11591 [05:17<2:09:52,  1.37it/s]
Iteration:   8%|▊         | 881/11591 [05:18<2:06:06,  1.42it/s]
Iteration:   8%|▊         | 882/11591 [05:18<2:02:09,  1.46it/s]
Iteration:   8%|▊         | 883/11591 [05:18<1:58:17,  1.51it/s]
Iteration:   8%|▊         | 884/11591 [05:19<1:55:55,  1.54it/s]
Iteration:   8%|▊         | 885/11591 [05:19<1:52:13,  1.59it/s]
Iteration:   8%|▊         | 886/11591 [05:19<1:48:56,  1.64it/s]
Iteration:   8%|▊         | 887/11591 [05:19<1:46:00,  1.68it/s]
Iteration:   8%|▊         | 888/11591 [05:20<1:43:12,  1.73it/s]
Iteration:   8%|▊         | 889/11591 [05:20<1:39:49,  1.79it/s]
Iteration:   8%|▊         | 890/11591 [05:20<1:36:36,  1.85it/s]
Iteration:   8%|▊         | 891/11591 [05:20<1:33:51,  1.90it/s]
Iteration:   8%|▊         | 892/11591 [05:21<1:30:56,  1.96it/s]
Iteration:   8%|▊         | 893/11591 [05:21<1:28:16,  2.02it/s]
Iteration:   8%|▊         | 894/11591 [05:21<1:26:28,  2.06it/s]
Iteration:   8%|▊         | 895/11591 [05:21<1:24:07,  2.12it/s]
Iteration:   8%|▊         | 896/11591 [05:21<1:21:54,  2.18it/s]
Iteration:   8%|▊         | 897/11591 [05:22<1:22:56,  2.15it/s]
Iteration:   8%|▊         | 898/11591 [05:25<1:48:17,  1.65it/s]
Iteration:   8%|▊         | 899/11591 [05:28<2:02:44,  1.45it/s]
Iteration:   8%|▊         | 900/11591 [05:28<1:58:42,  1.50it/s]
Iteration:   8%|▊         | 901/11591 [05:28<1:54:32,  1.56it/s]
Iteration:   8%|▊         | 902/11591 [05:29<1:59:11,  1.49it/s]
Iteration:   8%|▊         | 903/11591 [05:29<1:55:24,  1.54it/s]
Iteration:   8%|▊         | 904/11591 [05:30<1:58:09,  1.51it/s]
Iteration:   8%|▊         | 905/11591 [05:32<2:04:27,  1.43it/s]
Iteration:   8%|▊         | 906/11591 [05:32<1:59:47,  1.49it/s]
Iteration:   8%|▊         | 907/11591 [05:32<1:58:30,  1.50it/s]
Iteration:   8%|▊         | 908/11591 [05:34<2:04:50,  1.43it/s]
Iteration:   8%|▊         | 909/11591 [05:34<2:00:22,  1.48it/s]
Iteration:   8%|▊         | 910/11591 [05:35<2:02:09,  1.46it/s]
Iteration:   8%|▊         | 911/11591 [05:37<2:14:03,  1.33it/s]
Iteration:   8%|▊         | 912/11591 [05:41<2:41:40,  1.10it/s]
Iteration:   8%|▊         | 913/11591 [05:47<3:31:19,  1.19s/it]
Iteration:   8%|▊         | 914/11591 [05:48<3:22:23,  1.14s/it]
Iteration:   8%|▊         | 915/11591 [05:51<3:46:02,  1.27s/it]
Iteration:   8%|▊         | 916/11591 [05:52<3:36:46,  1.22s/it]
Iteration:   8%|▊         | 917/11591 [05:52<3:27:35,  1.17s/it]
Iteration:   8%|▊         | 918/11591 [06:01<4:38:54,  1.57s/it]
Iteration:   8%|▊         | 919/11591 [06:13<6:12:08,  2.09s/it]
Iteration:   8%|▊         | 920/11591 [06:23<7:22:35,  2.49s/it]
Iteration:   8%|▊         | 921/11591 [06:25<7:15:57,  2.45s/it]
Iteration:   8%|▊         | 922/11591 [06:26<7:08:05,  2.41s/it]
Iteration:   8%|▊         | 923/11591 [06:26<6:48:19,  2.30s/it]
Iteration:   8%|▊         | 924/11591 [06:37<8:01:41,  2.71s/it]
Iteration:   8%|▊         | 925/11591 [06:55<10:21:26,  3.50s/it]
Iteration:   8%|▊         | 926/11591 [06:57<10:05:29,  3.41s/it]
Iteration:   8%|▊         | 927/11591 [06:58<9:44:36,  3.29s/it] 
Iteration:   8%|▊         | 928/11591 [06:58<9:16:48,  3.13s/it]
Iteration:   8%|▊         | 929/11591 [07:03<9:33:20,  3.23s/it]
Iteration:   8%|▊         | 930/11591 [07:11<10:13:01,  3.45s/it]
Iteration:   8%|▊         | 931/11591 [07:11<9:44:22,  3.29s/it] 
Iteration:   8%|▊         | 932/11591 [07:12<9:24:25,  3.18s/it]
Iteration:   8%|▊         | 933/11591 [07:13<8:57:55,  3.03s/it]
Iteration:   8%|▊         | 934/11591 [07:15<8:55:50,  3.02s/it]
Iteration:   8%|▊         | 935/11591 [07:18<8:53:16,  3.00s/it]
Iteration:   8%|▊         | 936/11591 [07:21<8:55:14,  3.01s/it]
Iteration:   8%|▊         | 937/11591 [07:24<8:49:46,  2.98s/it]
Iteration:   8%|▊         | 938/11591 [07:37<10:21:38,  3.50s/it]
Iteration:   8%|▊         | 939/11591 [07:49<11:33:50,  3.91s/it]
Iteration:   8%|▊         | 940/11591 [08:06<13:30:50,  4.57s/it]
Iteration:   8%|▊         | 941/11591 [08:13<13:55:59,  4.71s/it]
Iteration:   8%|▊         | 942/11591 [08:15<13:27:44,  4.55s/it]
Iteration:   8%|▊         | 943/11591 [08:23<13:59:22,  4.73s/it]
Iteration:   8%|▊         | 944/11591 [08:23<13:19:22,  4.50s/it]
Iteration:   8%|▊         | 945/11591 [08:23<12:40:58,  4.29s/it]
Iteration:   8%|▊         | 946/11591 [08:26<12:29:28,  4.22s/it]
Iteration:   8%|▊         | 947/11591 [08:27<11:54:38,  4.03s/it]
Iteration:   8%|▊         | 948/11591 [08:46<14:10:46,  4.80s/it]
Iteration:   8%|▊         | 949/11591 [08:56<14:54:05,  5.04s/it]
Iteration:   8%|▊         | 950/11591 [08:56<14:11:09,  4.80s/it]
Iteration:   8%|▊         | 951/11591 [08:56<13:31:15,  4.57s/it]
Iteration:   8%|▊         | 952/11591 [08:56<12:52:26,  4.36s/it]
Iteration:   8%|▊         | 953/11591 [08:57<12:15:55,  4.15s/it]
Iteration:   8%|▊         | 954/11591 [08:57<11:41:52,  3.96s/it]
Iteration:   8%|▊         | 955/11591 [08:57<11:08:39,  3.77s/it]
Iteration:   8%|▊         | 956/11591 [08:57<10:37:18,  3.60s/it]
Iteration:   8%|▊         | 957/11591 [08:58<10:08:00,  3.43s/it]
Iteration:   8%|▊         | 958/11591 [08:58<9:39:33,  3.27s/it] 
Iteration:   8%|▊         | 959/11591 [08:58<9:12:45,  3.12s/it]
Iteration:   8%|▊         | 960/11591 [08:59<8:47:53,  2.98s/it]
Iteration:   8%|▊         | 961/11591 [08:59<8:23:33,  2.84s/it]
Iteration:   8%|▊         | 962/11591 [08:59<8:00:37,  2.71s/it]
Iteration:   8%|▊         | 963/11591 [08:59<7:39:10,  2.59s/it]
Iteration:   8%|▊         | 964/11591 [09:00<7:18:04,  2.47s/it]
Iteration:   8%|▊         | 965/11591 [09:00<6:58:14,  2.36s/it]
Iteration:   8%|▊         | 966/11591 [09:00<6:40:07,  2.26s/it]
Iteration:   8%|▊         | 967/11591 [09:00<6:22:19,  2.16s/it]
Iteration:   8%|▊         | 968/11591 [09:01<6:07:58,  2.08s/it]
Iteration:   8%|▊         | 969/11591 [09:08<6:49:17,  2.31s/it]
Iteration:   8%|▊         | 970/11591 [09:08<6:35:48,  2.24s/it]
Iteration:   8%|▊         | 971/11591 [09:09<6:19:19,  2.14s/it]
Iteration:   8%|▊         | 972/11591 [09:09<6:04:59,  2.06s/it]
Iteration:   8%|▊         | 973/11591 [09:10<5:55:05,  2.01s/it]
Iteration:   8%|▊         | 974/11591 [09:10<5:38:57,  1.92s/it]
Iteration:   8%|▊         | 975/11591 [09:17<6:21:58,  2.16s/it]
Iteration:   8%|▊         | 976/11591 [10:49<19:31:22,  6.62s/it]
Iteration:   8%|▊         | 977/11591 [11:19<23:03:14,  7.82s/it]
Iteration:   8%|▊         | 978/11591 [11:19<21:55:57,  7.44s/it]
Iteration:   8%|▊         | 979/11591 [11:23<21:19:53,  7.24s/it]
Iteration:   8%|▊         | 980/11591 [11:23<20:19:10,  6.89s/it]
Iteration:   8%|▊         | 981/11591 [11:24<19:21:02,  6.57s/it]
Iteration:   8%|▊         | 982/11591 [11:24<18:26:52,  6.26s/it]
Iteration:   8%|▊         | 983/11591 [11:24<17:34:50,  5.97s/it]
Iteration:   8%|▊         | 984/11591 [11:25<16:44:45,  5.68s/it]
Iteration:   8%|▊         | 985/11591 [11:25<15:57:48,  5.42s/it]
Iteration:   9%|▊         | 986/11591 [11:25<15:12:29,  5.16s/it]
Iteration:   9%|▊         | 987/11591 [11:26<14:30:01,  4.92s/it]
Iteration:   9%|▊         | 988/11591 [11:26<13:50:16,  4.70s/it]
Iteration:   9%|▊         | 989/11591 [11:27<13:11:51,  4.48s/it]
Iteration:   9%|▊         | 990/11591 [11:27<12:34:54,  4.27s/it]
Iteration:   9%|▊         | 991/11591 [11:27<11:59:16,  4.07s/it]
Iteration:   9%|▊         | 992/11591 [11:27<11:26:01,  3.88s/it]
Iteration:   9%|▊         | 993/11591 [11:28<10:53:44,  3.70s/it]
Iteration:   9%|▊         | 994/11591 [11:28<10:23:45,  3.53s/it]
Iteration:   9%|▊         | 995/11591 [11:30<10:08:31,  3.45s/it]
Iteration:   9%|▊         | 996/11591 [11:31<9:45:03,  3.31s/it] 
Iteration:   9%|▊         | 997/11591 [11:33<9:34:45,  3.26s/it]
Iteration:   9%|▊         | 998/11591 [11:33<9:08:10,  3.10s/it]
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [23:22<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [23:22<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [23:22<?, ?it/s]                   
Iteration:   9%|▊         | 999/11591 [23:22<8:42:51,  2.96s/it]

2022-11-22 21:25:27 - MSE evaluation (lower = better) on en-es-tsv_dev.tsv dataset in epoch 0 after 1000 steps:
2022-11-22 21:25:27 - MSE (*100):	0.080668
2022-11-22 21:25:27 - Evaluating translation matching Accuracy on en-es-tsv_dev.tsv dataset in epoch 0 after 1000 steps:

                                             
Epoch:   0%|          | 0/10 [27:15<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [27:15<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [27:15<?, ?it/s]                   
Iteration:   9%|▊         | 999/11591 [27:15<8:42:51,  2.96s/it]

2022-11-22 21:29:20 - Accuracy src2trg: 89.18
2022-11-22 21:29:20 - Accuracy trg2src: 88.20
2022-11-22 21:29:20 - EmbeddingSimilarityEvaluator: Evaluating the model on STS.en-en.txt dataset in epoch 0 after 1000 steps:

                                             
Epoch:   0%|          | 0/10 [27:22<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [27:22<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [27:22<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [27:22<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [27:22<?, ?it/s]                   
Iteration:   9%|▊         | 999/11591 [27:22<8:42:51,  2.96s/it]

2022-11-22 21:29:27 - Cosine-Similarity :	Pearson: 0.6608	Spearman: 0.7028
2022-11-22 21:29:27 - Manhattan-Distance:	Pearson: 0.6715	Spearman: 0.6777
2022-11-22 21:29:27 - Euclidean-Distance:	Pearson: 0.6700	Spearman: 0.6754
2022-11-22 21:29:27 - Dot-Product-Similarity:	Pearson: 0.5935	Spearman: 0.5990
2022-11-22 21:29:27 - EmbeddingSimilarityEvaluator: Evaluating the model on STS.es-en.txt dataset in epoch 0 after 1000 steps:

                                             
Epoch:   0%|          | 0/10 [27:23<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [27:23<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [27:23<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [27:23<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [27:23<?, ?it/s]                   
Iteration:   9%|▊         | 999/11591 [27:23<8:42:51,  2.96s/it]

2022-11-22 21:29:28 - Cosine-Similarity :	Pearson: 0.5448	Spearman: 0.5624
2022-11-22 21:29:28 - Manhattan-Distance:	Pearson: 0.5528	Spearman: 0.5367
2022-11-22 21:29:28 - Euclidean-Distance:	Pearson: 0.5532	Spearman: 0.5368
2022-11-22 21:29:28 - Dot-Product-Similarity:	Pearson: 0.4341	Spearman: 0.4397
2022-11-22 21:29:28 - EmbeddingSimilarityEvaluator: Evaluating the model on STS.es-es.txt dataset in epoch 0 after 1000 steps:

                                             
Epoch:   0%|          | 0/10 [27:23<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [27:23<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [27:23<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [27:23<?, ?it/s]                   
                                             8:42:51,  2.96s/it]
Epoch:   0%|          | 0/10 [27:23<?, ?it/s]                   
Iteration:   9%|▊         | 999/11591 [27:23<8:42:51,  2.96s/it]

2022-11-22 21:29:28 - Cosine-Similarity :	Pearson: 0.6925	Spearman: 0.7365
2022-11-22 21:29:28 - Manhattan-Distance:	Pearson: 0.7259	Spearman: 0.7173
2022-11-22 21:29:28 - Euclidean-Distance:	Pearson: 0.7246	Spearman: 0.7168
2022-11-22 21:29:28 - Dot-Product-Similarity:	Pearson: 0.5546	Spearman: 0.5806
2022-11-22 21:29:28 - Save model to output/make-multilingual-en-es-2022-11-22_20-59-30


print('Tiempo de entrenamiento: {} seg'.format(t2-t1))

Tiempo de entrenamiento: 34439.96579003334 seg


!mkdir Spanish_SBERT\modelos_entrenados\Spanish_SBERT_100_2

A subdirectory or file Spanish_SBERT\modelos_entrenados\Spanish_SBERT_100_2 already exists.


student_model.save('Spanish_SBERT/modelos_entrenados/Spanish_SBERT_100_2')

2022-11-23 06:36:05 - Save model to Spanish_SBERT/modelos_entrenados/Spanish_SBERT_100_2


%%capture

teacher_model_name = 'all-mpnet-base-v2'
student_model_name = 'bert-base-multilingual-cased' 

from sentence_transformers import SentenceTransformer, util

model_teacher = SentenceTransformer(teacher_model_name)
model_student_noT = SentenceTransformer(student_model_name)

Some weights of the model checkpoint at C:\Users\josep/.cache\torch\sentence_transformers\bert-base-multilingual-cased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias']
- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).


sent1_es = 'hola me gustaria abrir una cuenta en su banco'
sent1_en = 'hello i would like to open an account at your bank'

#sent2 = 'estimados quiero tener una cuenta corriente con ustedes'

code1 = model_teacher.encode(sent1_en)
code2_1 = model_student_noT.encode(sent1_en)
code2_2 = model_student_noT.encode(sent1_es)
code3_1 = student_model.encode(sent1_en)
code3_2 = student_model.encode(sent1_es)

Batches: 100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 21.91it/s]
Batches: 100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 67.20it/s]
Batches: 100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 69.52it/s]
Batches: 100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 69.52it/s]
Batches: 100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 69.52it/s]


util.cos_sim(code1, code3_2)*100

tensor([[40.8690]])


util.cos_sim(code1, code2_2)*100

tensor([[4.3812]])


util.cos_sim(code3_1, code3_2)

tensor([[0.9241]])


util.cos_sim(code2_1, code2_2)

tensor([[0.6296]])


from nltk.corpus import stopwords

spanish_stopwords = stopwords.words('spanish')


from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import CountVectorizer

def get_keywords(text, ngram, top):
    
    n_gram_range = (ngram, ngram)
    
    count = CountVectorizer( ngram_range = n_gram_range, stop_words = spanish_stopwords ).fit( [text] )
    candidates = count.get_feature_names()
    
    doc_embed = student_model.encode([text])
    can_embed = student_model.encode(candidates)
    
    distances = cosine_similarity( doc_embed, can_embed )
    keywords = [ candidates[index] for index in distances.argsort()[0][-top:] ]
    
    return keywords


text = 'tengo un problema con el uso de mi tarjeta_credito para usarla en el extranjero pero digame cuando uso de ella ya que se supone lo tengo activado hable'

get_keywords(text = text, ngram = 1, top = 6)

C:\Users\josep\anaconda3\envs\nlp-39\lib\site-packages\sklearn\utils\deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.
  warnings.warn(msg, category=FutureWarning)
Batches: 100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  8.54it/s]
Batches: 100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 28.80it/s]

['supone', 'activado', 'problema', 'uso', 'tarjeta_credito', 'usarla']


%%capture

embed1 = 'sentence-transformers/multi-qa-MiniLM-L6-cos-v1'
embed2 = 'sentence-transformers/all-MiniLM-L6-v2' 

from sentence_transformers import SentenceTransformer, util

model_embed1 = SentenceTransformer(embed1)
model_embed2 = SentenceTransformer(embed2)


target_languages

{'es'}


all_languages

['en', 'es']


##### Read cross-lingual Semantic Textual Similarity (STS) data ####
all_languages = list(set(list(source_languages)+list(target_languages)))
sts_data = {}

#Open the ZIP File of STS2017-extended.zip and check for which language combinations we have STS data
with zipfile.ZipFile(sts_corpus) as zip:
    filelist = zip.namelist()
    sts_files = []

    for i in range(len(all_languages)):
        for j in range(i, len(all_languages)):
            lang1 = all_languages[i]
            lang2 = all_languages[j]
            filepath = 'STS2017-extended/STS.en-en.txt'
            if filepath not in filelist:
                lang1, lang2 = lang2, lang1
                filepath = 'STS2017-extended/STS.en-en.txt'.format(lang1, lang2)

            if filepath in filelist:
                filename = os.path.basename(filepath)
                sts_data[filename] = {'sentences1': [], 'sentences2': [], 'scores': []}

                fIn = zip.open(filepath)
                for line in io.TextIOWrapper(fIn, 'utf8'):
                    sent1, sent2, score = line.strip().split("\t")
                    score = float(score)
                    sts_data[filename]['sentences1'].append(sent1)
                    sts_data[filename]['sentences2'].append(sent2)
                    sts_data[filename]['scores'].append(score)

#for filename, data in sts_data.items():
test_evaluator_en_en = evaluation.EmbeddingSimilarityEvaluator(data['sentences1'], data['sentences2'], data['scores'], batch_size=inference_batch_size, name=filename, show_progress_bar=False)
#    evaluators.append(test_evaluator)


##### Read cross-lingual Semantic Textual Similarity (STS) data ####
#all_languages = list(set(list(source_languages)+list(target_languages)))
all_languages = ['en', 'es']
sts_data = {}
1
sts_corpus = "Spanish_SBERT/datasets/STS2017-extended.zip"     # Extended STS2017 dataset for more languages

#Open the ZIP File of STS2017-extended.zip and check for which language combinations we have STS data
with zipfile.ZipFile(sts_corpus) as zip:
    filelist = zip.namelist()
    sts_files = []

    for i in range(len(all_languages)):
        for j in range(i, len(all_languages)):
            lang1 = all_languages[i]
            lang2 = all_languages[j]
            filepath = 'STS2017-extended/STS.es-es.txt'
            if filepath not in filelist:
                lang1, lang2 = lang2, lang1
                filepath = 'STS2017-extended/STS.es-es.txt'

            if filepath in filelist:
                filename = os.path.basename(filepath)
                sts_data[filename] = {'sentences1': [], 'sentences2': [], 'scores': []}

                fIn = zip.open(filepath)
                for line in io.TextIOWrapper(fIn, 'utf8'):
                    sent1, sent2, score = line.strip().split("\t")
                    score = float(score)
                    sts_data[filename]['sentences1'].append(sent1)
                    sts_data[filename]['sentences2'].append(sent2)
                    sts_data[filename]['scores'].append(score)

for filename, data in sts_data.items():
    test_evaluator_es_es = evaluation.EmbeddingSimilarityEvaluator(data['sentences1'], data['sentences2'], data['scores'], batch_size=64, name=filename, show_progress_bar=False)
#    evaluators.append(test_evaluator)


model_embed2.evaluate(
    evaluator: sentence_transformers.evaluation.SentenceEvaluator.SentenceEvaluator,
    output_path: str = None,
)


sentence_transformers.evaluation.SentenceEvaluator

sentence_transformers.evaluation.SentenceEvaluator.SentenceEvaluator


test_evaluator(model_embed2)

2022-11-18 13:13:42 - EmbeddingSimilarityEvaluator: Evaluating the model on STS.es-es.txt dataset:
2022-11-18 13:13:44 - Cosine-Similarity :	Pearson: 0.7704	Spearman: 0.7612
2022-11-18 13:13:44 - Manhattan-Distance:	Pearson: 0.7787	Spearman: 0.7565
2022-11-18 13:13:44 - Euclidean-Distance:	Pearson: 0.7822	Spearman: 0.7612
2022-11-18 13:13:44 - Dot-Product-Similarity:	Pearson: 0.7704	Spearman: 0.7611

0.7611764487426621


modelos_entrenados = [model_teacher,
                      model_student_noT,
                      student_model,]
                      #model_embed1,
                      #model_embed2]

modelos_entrenados_label = ['model_teacher',
                            'model_student_noT',
                            'student_model',]
                            #'model_embed1',
                            #'model_embed2']

for i in range( len(modelos_entrenados) ):
    print( modelos_entrenados_label[i] + ':' )
    print( test_evaluator(modelos_entrenados[i]) )
    print()

model_teacher:
2022-11-18 13:46:58 - EmbeddingSimilarityEvaluator: Evaluating the model on STS.es-es.txt dataset:
2022-11-18 13:46:59 - Cosine-Similarity :	Pearson: 0.7859	Spearman: 0.7840
2022-11-18 13:46:59 - Manhattan-Distance:	Pearson: 0.7960	Spearman: 0.7785
2022-11-18 13:46:59 - Euclidean-Distance:	Pearson: 0.7999	Spearman: 0.7840
2022-11-18 13:46:59 - Dot-Product-Similarity:	Pearson: 0.7859	Spearman: 0.7840
0.7840443150043068

model_student_noT:
2022-11-18 13:46:59 - EmbeddingSimilarityEvaluator: Evaluating the model on STS.es-es.txt dataset:
2022-11-18 13:46:59 - Cosine-Similarity :	Pearson: 0.5456	Spearman: 0.5669
2022-11-18 13:46:59 - Manhattan-Distance:	Pearson: 0.5727	Spearman: 0.5717
2022-11-18 13:46:59 - Euclidean-Distance:	Pearson: 0.5743	Spearman: 0.5737
2022-11-18 13:46:59 - Dot-Product-Similarity:	Pearson: 0.3576	Spearman: 0.3705
0.5737029785343253

student_model:
2022-11-18 13:46:59 - EmbeddingSimilarityEvaluator: Evaluating the model on STS.es-es.txt dataset:
2022-11-18 13:46:59 - Cosine-Similarity :	Pearson: 0.6931	Spearman: 0.7352
2022-11-18 13:46:59 - Manhattan-Distance:	Pearson: 0.7254	Spearman: 0.7170
2022-11-18 13:46:59 - Euclidean-Distance:	Pearson: 0.7242	Spearman: 0.7155
2022-11-18 13:46:59 - Dot-Product-Similarity:	Pearson: 0.5532	Spearman: 0.5806
0.7352274038094662


test_evaluator(modelos_entrenados[i])

zip

<zipfile.ZipFile [closed]>


#mkdir Spanish_SBERT\modelos_entrenados\Spanish_SBERT_100_2

A subdirectory or file Spanish_SBERT\modelos_entrenados\Spanish_SBERT_100_2 already exists.


from sentence_transformers import SentenceTransformer, util

model_load = SentenceTransformer('Spanish_SBERT/modelos_entrenados/Spanish_SBERT_100_2')


model_load

SentenceTransformer(
  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: BertModel 
  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
)


student_model

SentenceTransformer(
  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: BertModel 
  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
)


test_evaluator_es_es( model_load )

0.7352274038094662


student_model

SentenceTransformer(
  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: BertModel 
  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
)


from multiprocessing import Process

count1 = 0
count2 = 0

def func1():
    print('func1: starting')
    for i in xrange(10000000): count1 += i
    print('func1: finishing')

def func2():
    print ('func2: starting')
    for i in xrange(19999999): count2 += i
    print ('func2: finishing')

if __name__ == '__main__':
    p1 = Process(target=func1)
    p1.start()
    p2 = Process(target=func2)
    p2.start()
    p1.join()
    p2.join()


count1

0


count1 = 0
count2 = 0


# Import module
import threading
import time
  
# A sample function to print squares
def print_squares(thread_name, numbers):
    
    for number in numbers:
        print(thread_name, number**2)
          
        # Produce some delay to see the output
        time.sleep(1)
  
  
# Creating 3 threads that execute the same 
# function with different parameters
thread1 = threading.Thread(target=print_squares, 
                           args=("thread1", [1, 2, 3, 4, 5]))
  
thread2 = threading.Thread(target=print_squares, 
                           args=("thread2", [6, 7, 8, 9, 10]))
  
thread3 = threading.Thread(target=print_squares, 
                           args=("thread3", [11, 12, 13, 14, 15]))
  
# Start the threads
thread1.start()
thread2.start()
thread3.start()
  
# Join the threads before 
# moving further
thread1.join()
thread2.join()
thread3.join()

thread1 1
thread2 36
thread3 121
thread1thread2 49
 4
thread3 144
thread2thread1 9
thread3 169
 64
thread3thread1 16
thread2 81
 196
thread1thread2 100
 25
thread3 225


embed1 = 'Spanish_SBERT/modelos_entrenados/Spanish_SBERT_100_2' 

from sentence_transformers import SentenceTransformer, util

model_embed1 = SentenceTransformer(embed1)


model_embed1

SentenceTransformer(
  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: BertModel 
  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
)


import tarfile
import subprocess

subprocess.call( ["tar", "-czf", "New_Embed_model.tar.gz", "Spanish_SBERT"] )

0

	source	target
0	Thank you so much, Chris.	Muchas gracias Chris.
1	And it's truly a great honor to have the oppor...	Y es en verdad un gran honor tener la oportuni...
2	I have been blown away by this conference, and...	He quedado conmovido por esta conferencia, y d...
3	And I say that sincerely, partly because (Mock...	Y digo eso sinceramente, en parte porque -- (S...
4	(Laughter) Put yourselves in my position.	(Risas) ¡Pónganse en mi posición!
...	...	...
405950	It was indeed true that the stories of history...	Es cierto que a la historia la contaban los qu...
405951	My work speaks for those who will no longer le...	y mi trabajo habla por aquellos que ya no van ...
405952	Today, we stand ready to tell our own stories ...	Hoy, estamos parados, listos para contar nuest...
405953	But the question still remains: are you prepar...	Pero la pregunta sigue en pie: ¿Están preparad...
405954	I hope you are, because we are coming regardless.	Espero que lo estén, porque allá vamos de todo...

	source	target
0	thank you so much chris	muchas gracias chris
1	and its truly a great honor to have the opport...	y es en verdad un gran honor tener la oportuni...
2	i have been blown away by this conference and ...	he quedado conmovido por esta conferencia y de...
3	and i say that sincerely partly because mock s...	y digo eso sinceramente en parte porque solloz...
4	laughter put yourselves in my position	risas ponganse en mi posicion
...	...	...
405950	it was indeed true that the stories of history...	es cierto que a la historia la contaban los qu...
405951	my work speaks for those who will no longer le...	y mi trabajo habla por aquellos que ya no van ...
405952	today we stand ready to tell our own stories w...	hoy estamos parados listos para contar nuestra...
405953	but the question still remains are you prepare...	pero la pregunta sigue en pie estan preparados...
405954	i hope you are because we are coming regardless	espero que lo esten porque alla vamos de todos...

	source	target
0	and its meant to kind of reflect how we engage...	y esto pretende reflejar de algun modo como no...
1	so there are two big issues here	asi que hay dos grandes problemas
2	adorno used to say "intolerance of ambiguity i...	adorno solia decir "la intolerancia a la ambig...
3	so we walked around the backside to see what w...	entonces caminamos a la parte de atras para ve...
4	i just cant get over this that youve got these...	simplemente no dejo de impresionarme por esto ...

	source	target
0	psychologists medical scientists economists we...	psicologos investigadores medicos y economista...
1	and you have to understand this is not how it ...	y hay que entender que esta no es la manera co...
2	in tuscaloosa at the university of alabama the...	en tuscaloosa en la universidad de alabama hay...
3	then she said "the second thing i want you to ...	luego me dijo "lo segundo que quiero que me pr...
4	white objects reflect all colors of light whil...	los objetos blancos reflejan todos los colores...

Separamos en entrenamiento y testeo¶

SBERT Spanish¶